From a5d7fc1d1000ffb1d21796f5d587f277c2957d66 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Thu, 21 Mar 2024 12:30:48 +0900 Subject: [PATCH 001/933] [mlir][sparse] Fix typos in comments (#86074) --- .../test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir | 3 +-- .../Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir index 34d450c2403f..7ecccad212cd 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir @@ -285,8 +285,7 @@ module { %has_runtime = sparse_tensor.has_runtime_library scf.if %has_runtime { // sparse_tensor.assemble copies buffers when running with the runtime - // library. Deallocations are needed not needed when running in codgen - // mode. + // library. Deallocations are not needed when running in codegen mode. bufferization.dealloc_tensor %s4 : tensor<10x10xf64, #SortedCOO> bufferization.dealloc_tensor %s5 : tensor<10x10xf64, #SortedCOOI32> bufferization.dealloc_tensor %csr : tensor<2x2xf64, #CSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir index fe8836266a47..20ae7e86285c 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir @@ -146,8 +146,7 @@ module { %has_runtime = sparse_tensor.has_runtime_library scf.if %has_runtime { // sparse_tensor.assemble copies buffers when running with the runtime - // library. Deallocations are needed not needed when running in codgen - // mode. + // library. Deallocations are not needed when running in codegen mode. bufferization.dealloc_tensor %s0 : tensor<4x3x2xf32, #CCC> bufferization.dealloc_tensor %s1 : tensor<4x3x2xf32, #BatchedCSR> bufferization.dealloc_tensor %s2 : tensor<4x3x2xf32, #CSRDense> -- GitLab From 7bb87d533891c2bcfa1c9132605f0d3e8227d444 Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar Date: Thu, 21 Mar 2024 09:54:03 +0530 Subject: [PATCH 002/933] [AArch64][GlobalISel] Take abs scalar codegen closer to SDAG (#84886) This patch improves codegen for scalar (<128bits) version of llvm.abs intrinsic by using the existing non-XOR based lowering. This takes the generated code closer to SDAG. codegen with GISel for > 128 bit types is not very good with these method so not doing so. --- .../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 15 +++++++++- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +++++ .../AArch64/GlobalISel/legalize-abs.mir | 28 ++++++++++++------- llvm/test/CodeGen/AArch64/abs.ll | 26 ++++++++--------- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 5bb3692f0a46..284f434fbb9b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -429,6 +429,7 @@ public: LegalizeResult lowerDIVREM(MachineInstr &MI); LegalizeResult lowerAbsToAddXor(MachineInstr &MI); LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); + LegalizeResult lowerAbsToCNeg(MachineInstr &MI); LegalizeResult lowerVectorReduction(MachineInstr &MI); LegalizeResult lowerMemcpyInline(MachineInstr &MI); LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index abe23af00a78..8d608f6ac5e4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -8215,9 +8215,22 @@ LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) { // %res = G_SMAX %a, %v2 Register SrcReg = MI.getOperand(1).getReg(); LLT Ty = MRI.getType(SrcReg); + auto Zero = MIRBuilder.buildConstant(Ty, 0); + auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg); + MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) { + Register SrcReg = MI.getOperand(1).getReg(); + Register DestReg = MI.getOperand(0).getReg(); + LLT Ty = MRI.getType(SrcReg), IType = LLT::scalar(1); auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0); auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0); - MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); + auto ICmp = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, IType, SrcReg, Zero); + MIRBuilder.buildSelect(DestReg, ICmp, SrcReg, Sub); MI.eraseFromParent(); return Legalized; } diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 996abe8e4739..34e2c1d9c8e2 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1012,6 +1012,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) ABSActions .legalFor({s32, s64}); ABSActions.legalFor(PackedVectorAllTypeList) + .customIf([=](const LegalityQuery &Q) { + // TODO: Fix suboptimal codegen for 128+ bit types. + LLT SrcTy = Q.Types[0]; + return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128; + }) .widenScalarIf( [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; }, [=](const LegalityQuery &Query) { return std::make_pair(0, v4s16); }) @@ -1264,6 +1269,8 @@ bool AArch64LegalizerInfo::legalizeCustom( return legalizeDynStackAlloc(MI, Helper); case TargetOpcode::G_PREFETCH: return legalizePrefetch(MI, Helper); + case TargetOpcode::G_ABS: + return Helper.lowerAbsToCNeg(MI); } llvm_unreachable("expected switch to return"); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir index 3123e304116f..0d429ae38402 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir @@ -8,11 +8,12 @@ body: | bb.0: ; CHECK-LABEL: name: abs_s32 ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31 - ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s64) - ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[ASHR]] - ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ADD]], [[ASHR]] - ; CHECK-NEXT: $w0 = COPY [[XOR]](s32) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY]] + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s32), [[C]] + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[SUB]] + ; CHECK-NEXT: $w0 = COPY [[SELECT]](s32) + ; ; CHECK-CSSC-LABEL: name: abs_s32 ; CHECK-CSSC: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 ; CHECK-CSSC-NEXT: [[ABS:%[0-9]+]]:_(s32) = G_ABS [[COPY]] @@ -28,11 +29,12 @@ body: | bb.0: ; CHECK-LABEL: name: abs_s64 ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63 - ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64) - ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[ASHR]] - ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[ADD]], [[ASHR]] - ; CHECK-NEXT: $x0 = COPY [[XOR]](s64) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[C]], [[COPY]] + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s64), [[C]] + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s32), [[COPY]], [[SUB]] + ; CHECK-NEXT: $x0 = COPY [[SELECT]](s64) + ; ; CHECK-CSSC-LABEL: name: abs_s64 ; CHECK-CSSC: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 ; CHECK-CSSC-NEXT: [[ABS:%[0-9]+]]:_(s64) = G_ABS [[COPY]] @@ -55,6 +57,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<4 x s16>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<4 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s16 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -82,6 +85,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<8 x s16>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v8s16 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} @@ -109,6 +113,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<2 x s32>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<2 x s32>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v2s32 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -136,6 +141,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<4 x s32>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<4 x s32>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s32 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} @@ -163,6 +169,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<8 x s8>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<8 x s8>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s8 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -190,6 +197,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<16 x s8>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<16 x s8>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v16s8 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AArch64/abs.ll b/llvm/test/CodeGen/AArch64/abs.ll index e00f70b94e3b..78c1ff7b9937 100644 --- a/llvm/test/CodeGen/AArch64/abs.ll +++ b/llvm/test/CodeGen/AArch64/abs.ll @@ -15,9 +15,8 @@ define i8 @abs_i8(i8 %a){ ; CHECK-GI-LABEL: abs_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: sxtb w8, w0 -; CHECK-GI-NEXT: asr w8, w8, #7 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i8 @llvm.abs.i8(i8 %a, i1 0) @@ -36,9 +35,8 @@ define i16 @abs_i16(i16 %a){ ; CHECK-GI-LABEL: abs_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: sxth w8, w0 -; CHECK-GI-NEXT: asr w8, w8, #15 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i16 @llvm.abs.i16(i16 %a, i1 0) @@ -55,9 +53,8 @@ define i32 @abs_i32(i32 %a){ ; ; CHECK-GI-LABEL: abs_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: asr w8, w0, #31 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i32 @llvm.abs.i32(i32 %a, i1 0) @@ -74,9 +71,8 @@ define i64 @abs_i64(i64 %a){ ; ; CHECK-GI-LABEL: abs_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: asr x8, x0, #63 -; CHECK-GI-NEXT: add x9, x0, x8 -; CHECK-GI-NEXT: eor x0, x9, x8 +; CHECK-GI-NEXT: cmp x0, #0 +; CHECK-GI-NEXT: cneg x0, x0, le ; CHECK-GI-NEXT: ret entry: %res = call i64 @llvm.abs.i64(i64 %a, i1 0) @@ -248,9 +244,9 @@ define <1 x i32> @abs_v1i32(<1 x i32> %a){ ; CHECK-GI-LABEL: abs_v1i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: asr w9, w8, #31 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: eor w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w8, w9, le ; CHECK-GI-NEXT: fmov s0, w8 ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-GI-NEXT: ret -- GitLab From 35d3b3430eff16403d004d9f0b0369f0814cf140 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Thu, 21 Mar 2024 14:16:02 +0900 Subject: [PATCH 003/933] [mlir][bufferization] Add "bottom-up from terminators" analysis heuristic (#83964) One-Shot Bufferize currently does not support loops where a yielded value bufferizes to a buffer that is different from the buffer of the region iter_arg. In such a case, the bufferization fails with an error such as: ``` Yield operand #0 is not equivalent to the corresponding iter bbArg scf.yield %0 : tensor<5xf32> ``` One common reason for non-equivalent buffers is that an op on the path from the region iter_arg to the terminator bufferizes out-of-place. Ops that are analyzed earlier are more likely to bufferize in-place. This commit adds a new heuristic that gives preference to ops that are reachable on the reverse SSA use-def chain from a region terminator and are within the parent region of the terminator. This is expected to work better than the existing heuristics for loops where an iter_arg is written to multiple times within a loop, but only one write is fed into the terminator. Current users of One-Shot Bufferize are not affected by this change. "Bottom-up" is still the default heuristic. Users can switch to the new heuristic manually. This commit also turns the "fuzzer" pass option into a heuristic, cleaning up the code a bit. --- .../IR/BufferizableOpInterface.h | 4 - .../Transforms/OneShotAnalysis.h | 12 +- .../Bufferization/Transforms/Passes.td | 18 +++ .../Bufferization/Transforms/Bufferize.cpp | 5 + .../Transforms/OneShotAnalysis.cpp | 116 ++++++++++++++---- .../Dialect/Arith/one-shot-bufferize.mlir | 6 +- ...ne-shot-bufferize-allow-return-allocs.mlir | 6 +- ...e-analysis-bottom-up-from-terminators.mlir | 36 ++++++ .../one-shot-bufferize-partial.mlir | 6 +- .../Transforms/one-shot-bufferize.mlir | 6 +- ...-module-bufferize-allow-return-allocs.mlir | 6 +- .../one-shot-module-bufferize-analysis.mlir | 11 +- .../Transforms/one-shot-module-bufferize.mlir | 6 +- .../Dialect/Linalg/one-shot-bufferize.mlir | 6 +- .../SCF/one-shot-bufferize-analysis.mlir | 6 +- mlir/test/Dialect/SCF/one-shot-bufferize.mlir | 6 +- .../Dialect/Tensor/one-shot-bufferize.mlir | 6 +- 17 files changed, 198 insertions(+), 64 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h index 3a61a4b34765..2d8add82383b 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h @@ -366,10 +366,6 @@ struct BufferizationOptions { DefaultMemorySpaceFn defaultMemorySpaceFn = [](TensorType t) -> std::optional { return Attribute(); }; - /// Seed for the analysis fuzzer. If set to `0`, the fuzzer is deactivated. - /// Should be used only with `testAnalysisOnly = true`. - unsigned analysisFuzzerSeed = 0; - /// If set to `true`, the analysis is skipped. A buffer is copied before every /// write. This flag cannot be used together with `testAnalysisOnly = true`. bool copyBeforeWrite = false; diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h index a29af853eb21..d50a3042aeea 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h @@ -24,7 +24,12 @@ class OneShotAnalysisState; /// Options for analysis-enabled bufferization. struct OneShotBufferizationOptions : public BufferizationOptions { - enum class AnalysisHeuristic { BottomUp, TopDown }; + enum class AnalysisHeuristic { + BottomUp, + TopDown, + BottomUpFromTerminators, + Fuzzer + }; OneShotBufferizationOptions() = default; @@ -42,6 +47,11 @@ struct OneShotBufferizationOptions : public BufferizationOptions { /// Specify the functions that should not be analyzed. copyBeforeWrite will be /// set to true when bufferizing them. llvm::ArrayRef noAnalysisFuncFilter; + + /// Seed for the analysis fuzzer. Used only if the heuristic is set to + /// `AnalysisHeuristic::Fuzzer`. The fuzzer should be used only with + /// `testAnalysisOnly = true`. + unsigned analysisFuzzerSeed = 0; }; /// State for analysis-enabled bufferization. This class keeps track of alias diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td index 1c3cdec81a39..1303dc2c9ae1 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td @@ -459,6 +459,24 @@ def OneShotBufferize : Pass<"one-shot-bufferize", "ModuleOp"> { argument is read/written and which returned values are aliasing/equivalent. For debugging purposes, such information can be printed with `test-analysis-only`. + + The order in which ops are analyzed is important. The analysis is greedy and + ops that are analyzed earlier are more likely to bufferize in-place. The + heuristic can be set with `analysis-heuristic`. At the moment, the following + heuristics are available: + + * `bottom-up` (default): Analyze ops from bottom to top. + * `top-down`: Analyze ops from top to bottom. + * `fuzzer`: Randomize the ordering of ops with `analysis-fuzzer-seed`. + * `bottom-up-from-terminators`: Traverse the reverse use-def chains of + tensor IR, starting from region branch terminators (bottom-up). Nested + regions are traversed before enclosing regions. Analyze the traversed ops + first, then analyze the remaining ops bottom-up. This heuristic is useful + for bufferizing loop constructs. One-Shot Bufferize currently supports + only such IR where yielded tensor values bufferize to equivalent region + iter_args, and first analyzing all ops on the path from the "yielding" op + to the beginning of the loop body makes it more likely for the region + iter_args and yielded values to bufferize to equivalent buffers. }]; let options = [ Option<"allowReturnAllocsFromLoops", "allow-return-allocs-from-loops", diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp index 8dbf70162012..32f4e6a0fe89 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp @@ -182,6 +182,11 @@ parseHeuristicOption(const std::string &s) { return OneShotBufferizationOptions::AnalysisHeuristic::BottomUp; if (s == "top-down") return OneShotBufferizationOptions::AnalysisHeuristic::TopDown; + if (s == "bottom-up-from-terminators") + return OneShotBufferizationOptions::AnalysisHeuristic:: + BottomUpFromTerminators; + if (s == "fuzzer") + return OneShotBufferizationOptions::AnalysisHeuristic::Fuzzer; llvm_unreachable("invalid analysisheuristic option"); } diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp index fba9cd873063..531016130d1d 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp @@ -51,6 +51,7 @@ #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/IR/AsmState.h" #include "mlir/IR/Dominance.h" +#include "mlir/IR/Iterators.h" #include "mlir/IR/Operation.h" #include "mlir/IR/TypeUtilities.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" @@ -1094,41 +1095,104 @@ static void equivalenceAnalysis(Operation *op, OneShotAnalysisState &state) { equivalenceAnalysis(ops, state); } -LogicalResult OneShotAnalysisState::analyzeOp(Operation *op, - const DominanceInfo &domInfo) { - // Collect ops so we can build our own reverse traversal. - SmallVector ops; - op->walk([&](Operation *op) { - // No tensors => no buffers. - if (!hasTensorSemantics(op)) +/// "Bottom-up from terminators" heuristic. +static SmallVector +bottomUpFromTerminatorsHeuristic(Operation *op, + const OneShotAnalysisState &state) { + SetVector traversedOps; + + // Find region terminators. + op->walk([&](RegionBranchTerminatorOpInterface term) { + if (!traversedOps.insert(term)) return; - ops.push_back(op); + // Follow the reverse SSA use-def chain from each yielded value as long as + // we stay within the same region. + SmallVector worklist; + for (Value v : term->getOperands()) { + if (!isa(v.getType())) + continue; + auto opResult = dyn_cast(v); + if (!opResult) + continue; + worklist.push_back(opResult); + } + while (!worklist.empty()) { + OpResult opResult = worklist.pop_back_val(); + Operation *defOp = opResult.getDefiningOp(); + if (!traversedOps.insert(defOp)) + continue; + if (!term->getParentRegion()->findAncestorOpInRegion(*defOp)) + continue; + AliasingOpOperandList aliases = state.getAliasingOpOperands(opResult); + for (auto alias : aliases) { + Value v = alias.opOperand->get(); + if (!isa(v.getType())) + continue; + auto opResult = dyn_cast(v); + if (!opResult) + continue; + worklist.push_back(opResult); + } + } }); - if (getOptions().analysisFuzzerSeed) { - // This is a fuzzer. For testing purposes only. Randomize the order in which - // operations are analyzed. The bufferization quality is likely worse, but - // we want to make sure that no assertions are triggered anywhere. - std::mt19937 g(getOptions().analysisFuzzerSeed); - llvm::shuffle(ops.begin(), ops.end(), g); - } + // Analyze traversed ops, then all remaining ops. + SmallVector result(traversedOps.begin(), traversedOps.end()); + op->walk([&](Operation *op) { + if (!traversedOps.contains(op) && hasTensorSemantics(op)) + result.push_back(op); + }); + return result; +} +LogicalResult OneShotAnalysisState::analyzeOp(Operation *op, + const DominanceInfo &domInfo) { OneShotBufferizationOptions::AnalysisHeuristic heuristic = getOptions().analysisHeuristic; - if (heuristic == OneShotBufferizationOptions::AnalysisHeuristic::BottomUp) { - // Default: Walk ops in reverse for better interference analysis. - for (Operation *op : reverse(ops)) - if (failed(analyzeSingleOp(op, domInfo))) - return failure(); - } else if (heuristic == - OneShotBufferizationOptions::AnalysisHeuristic::TopDown) { - for (Operation *op : ops) - if (failed(analyzeSingleOp(op, domInfo))) - return failure(); + + SmallVector orderedOps; + if (heuristic == + OneShotBufferizationOptions::AnalysisHeuristic::BottomUpFromTerminators) { + orderedOps = bottomUpFromTerminatorsHeuristic(op, *this); } else { - llvm_unreachable("unsupported heuristic"); + op->walk([&](Operation *op) { + // No tensors => no buffers. + if (!hasTensorSemantics(op)) + return; + orderedOps.push_back(op); + }); + switch (heuristic) { + case OneShotBufferizationOptions::AnalysisHeuristic::BottomUp: { + // Default: Walk ops in reverse for better interference analysis. + std::reverse(orderedOps.begin(), orderedOps.end()); + break; + } + case OneShotBufferizationOptions::AnalysisHeuristic::TopDown: { + // Ops are already sorted top-down in `orderedOps`. + break; + } + case OneShotBufferizationOptions::AnalysisHeuristic::Fuzzer: { + assert(getOptions().analysisFuzzerSeed && + "expected that fuzzer seed it set"); + // This is a fuzzer. For testing purposes only. Randomize the order in + // which operations are analyzed. The bufferization quality is likely + // worse, but we want to make sure that no assertions are triggered + // anywhere. + std::mt19937 g(getOptions().analysisFuzzerSeed); + llvm::shuffle(orderedOps.begin(), orderedOps.end(), g); + break; + } + default: { + llvm_unreachable("unsupported heuristic"); + } + } } + // Analyze ops in the computed order. + for (Operation *op : orderedOps) + if (failed(analyzeSingleOp(op, domInfo))) + return failure(); + equivalenceAnalysis(op, *this); return success(); } diff --git a/mlir/test/Dialect/Arith/one-shot-bufferize.mlir b/mlir/test/Dialect/Arith/one-shot-bufferize.mlir index 174bf2fc8e4b..f6bdca7f4d9e 100644 --- a/mlir/test/Dialect/Arith/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Arith/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir index e4375950d336..8f0170b17381 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // CHECK-LABEL: func @buffer_not_deallocated( // CHECK-SAME: %[[t:.*]]: tensor diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir new file mode 100644 index 000000000000..1b75edc4c157 --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir @@ -0,0 +1,36 @@ +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=bottom-up-from-terminators" -split-input-file | FileCheck %s + +// CHECK-LABEL: func @simple_test( +func.func @simple_test(%lb: index, %ub: index, %step: index, %f1: f32, %f2: f32) -> (tensor<5xf32>, tensor<5xf32>) { + %c0 = arith.constant 0 : index + %p = arith.constant 0.0 : f32 + + // Make sure that ops that feed into region terminators bufferize in-place + // (if possible). + // Note: This test case fails to bufferize with a "top-down" or "bottom-up" + // heuristic. + + %0 = tensor.empty() : tensor<5xf32> + %1 = scf.for %iv = %lb to %ub step %step iter_args(%t = %0) -> (tensor<5xf32>) { + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "false"]} + %2 = linalg.fill ins(%f1 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "true"]} + %3 = linalg.fill ins(%f2 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + %4 = vector.transfer_read %2[%c0], %p : tensor<5xf32>, vector<5xf32> + vector.print %4 : vector<5xf32> + scf.yield %3 : tensor<5xf32> + } + + %5 = tensor.empty() : tensor<5xf32> + %6 = scf.for %iv = %lb to %ub step %step iter_args(%t = %0) -> (tensor<5xf32>) { + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "true"]} + %7 = linalg.fill ins(%f1 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "false"]} + %8 = linalg.fill ins(%f2 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + %9 = vector.transfer_read %8[%c0], %p : tensor<5xf32>, vector<5xf32> + vector.print %9 : vector<5xf32> + scf.yield %7 : tensor<5xf32> + } + + return %1, %6 : tensor<5xf32>, tensor<5xf32> +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir index 2c5f2083f589..9380c81ce235 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir @@ -4,9 +4,9 @@ // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-unknown-ops unknown-type-conversion=identity-layout-map" -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP // Run fuzzer with different seeds. -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="dialect-filter=tensor,bufferization allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s --check-prefix=CHECK-TENSOR // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="dialect-filter=scf,bufferization allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s --check-prefix=CHECK-SCF diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir index 611b67e198c0..0ed3a9f077ce 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops" -verify-diagnostics -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -verify-diagnostics -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -verify-diagnostics -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -verify-diagnostics -split-input-file -o /dev/null // Run with top-down analysis. // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops analysis-heuristic=top-down" -verify-diagnostics -split-input-file | FileCheck %s --check-prefix=CHECK-TOP-DOWN-ANALYSIS diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir index 9319ac61d928..c58b153d438c 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir @@ -2,9 +2,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 " -split-input-file | FileCheck %s --check-prefix=NO-DROP // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir index 6e7b113aa35c..42d9cc00d3ff 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir @@ -1,9 +1,14 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null + +// Try different heuristics. Not checking the result, just make sure that we do +// not crash. +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=bottom-up-from-terminators" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=top-down" -split-input-file -o /dev/null // TODO: Extract op-specific test cases and move them to their respective // dialects. diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir index 39f4835b28ff..429c9e4dea9e 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir @@ -2,9 +2,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1" -canonicalize -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map" -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP diff --git a/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir b/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir index c69701b65e20..9616a3e32a06 100644 --- a/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -canonicalize -buffer-loop-hoisting -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -drop-equivalent-buffer-results -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP diff --git a/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir index 7d23498f32e1..4d82021e86f5 100644 --- a/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir +++ b/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // CHECK-LABEL: func @scf_for_yield_only func.func @scf_for_yield_only( diff --git a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir index 24da8d84b18e..485fdd9b0e59 100644 --- a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries" -cse -canonicalize -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir b/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir index 38c3bb8af810..e2169fe1404c 100644 --- a/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null -- GitLab From 733640d29ede70585e0e3e1dcc47b935981f791e Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 10:54:03 +0530 Subject: [PATCH 004/933] Attributor: Handle inferring align from use by atomics (#85762) --- .../Transforms/IPO/AttributorAttributes.cpp | 6 + llvm/test/Transforms/Attributor/align.ll | 163 +++++++++++++++--- .../test/Transforms/Attributor/nocapture-1.ll | 12 +- llvm/test/Transforms/Attributor/nofpclass.ll | 2 +- 4 files changed, 151 insertions(+), 32 deletions(-) diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index f98833bd1198..ff680e998e71 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -5190,6 +5190,12 @@ static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA, } else if (auto *LI = dyn_cast(I)) { if (LI->getPointerOperand() == UseV) MA = LI->getAlign(); + } else if (auto *AI = dyn_cast(I)) { + if (AI->getPointerOperand() == UseV) + MA = AI->getAlign(); + } else if (auto *AI = dyn_cast(I)) { + if (AI->getPointerOperand() == UseV) + MA = AI->getAlign(); } if (!MA || *MA <= QueryingAA.getKnownAlign()) diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll index 5103b6f1f1e9..9880e53fd43a 100644 --- a/llvm/test/Transforms/Attributor/align.ll +++ b/llvm/test/Transforms/Attributor/align.ll @@ -11,10 +11,10 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" ; TEST 1 ;; ;. -; CHECK: @[[A1:[a-zA-Z0-9_$"\\.-]+]] = common global i8 0, align 8 -; CHECK: @[[A2:[a-zA-Z0-9_$"\\.-]+]] = common global i8 0, align 16 -; CHECK: @[[CND:[a-zA-Z0-9_$"\\.-]+]] = external global i1 -; CHECK: @[[G:[a-zA-Z0-9_$"\\.-]+]] = global i8 0, align 32 +; CHECK: @a1 = common global i8 0, align 8 +; CHECK: @a2 = common global i8 0, align 16 +; CHECK: @cnd = external global i1 +; CHECK: @G = global i8 0, align 32 ;. define ptr @test1(ptr align 8 %0) #0 { ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable @@ -158,18 +158,31 @@ define internal ptr @f1(ptr readnone %0) local_unnamed_addr #0 { ; Function Attrs: nounwind readnone ssp uwtable define ptr @f2(ptr readnone %0) local_unnamed_addr #0 { -; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable -; CHECK-LABEL: define {{[^@]+}}@f2 -; CHECK-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { -; CHECK-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null -; CHECK-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] -; CHECK: 3: -; CHECK-NEXT: br label [[TMP5:%.*]] -; CHECK: 4: -; CHECK-NEXT: br label [[TMP5]] -; CHECK: 5: -; CHECK-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] -; CHECK-NEXT: ret ptr [[TMP6]] +; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable +; TUNIT-LABEL: define {{[^@]+}}@f2 +; TUNIT-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { +; TUNIT-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; TUNIT-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] +; TUNIT: 3: +; TUNIT-NEXT: br label [[TMP5:%.*]] +; TUNIT: 4: +; TUNIT-NEXT: br label [[TMP5]] +; TUNIT: 5: +; TUNIT-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] +; TUNIT-NEXT: ret ptr [[TMP6]] +; +; CGSCC: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable +; CGSCC-LABEL: define {{[^@]+}}@f2 +; CGSCC-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { +; CGSCC-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; CGSCC-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] +; CGSCC: 3: +; CGSCC-NEXT: br label [[TMP5:%.*]] +; CGSCC: 4: +; CGSCC-NEXT: br label [[TMP5]] +; CGSCC: 5: +; CGSCC-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] +; CGSCC-NEXT: ret ptr [[TMP6]] ; %2 = icmp eq ptr %0, null br i1 %2, label %5, label %3 @@ -222,7 +235,7 @@ define align 4 ptr @test7() #0 { ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable ; CGSCC-LABEL: define {{[^@]+}}@test7 ; CGSCC-SAME: () #[[ATTR1:[0-9]+]] { -; CGSCC-NEXT: [[C:%.*]] = tail call noundef nonnull align 8 dereferenceable(1) ptr @f1() #[[ATTR14:[0-9]+]] +; CGSCC-NEXT: [[C:%.*]] = tail call noundef nonnull align 8 dereferenceable(1) ptr @f1() #[[ATTR15:[0-9]+]] ; CGSCC-NEXT: ret ptr [[C]] ; %c = tail call ptr @f1(ptr align 8 dereferenceable(1) @a1) @@ -933,7 +946,7 @@ define i32 @musttail_caller_1(ptr %p) { ; TUNIT-NEXT: [[C:%.*]] = load i1, ptr @cnd, align 1 ; TUNIT-NEXT: br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]] ; TUNIT: mt: -; TUNIT-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef readonly [[P]]) #[[ATTR12:[0-9]+]] +; TUNIT-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef readonly [[P]]) #[[ATTR13:[0-9]+]] ; TUNIT-NEXT: ret i32 [[V]] ; TUNIT: exit: ; TUNIT-NEXT: ret i32 0 @@ -944,7 +957,7 @@ define i32 @musttail_caller_1(ptr %p) { ; CGSCC-NEXT: [[C:%.*]] = load i1, ptr @cnd, align 1 ; CGSCC-NEXT: br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]] ; CGSCC: mt: -; CGSCC-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P]]) #[[ATTR15:[0-9]+]] +; CGSCC-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P]]) #[[ATTR16:[0-9]+]] ; CGSCC-NEXT: ret i32 [[V]] ; CGSCC: exit: ; CGSCC-NEXT: ret i32 0 @@ -1076,13 +1089,13 @@ define ptr @aligned_8_return_caller(ptr align(16) %a, i1 %c1, i1 %c2) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none) ; TUNIT-LABEL: define {{[^@]+}}@aligned_8_return_caller ; TUNIT-SAME: (ptr nofree readnone align 16 "no-capture-maybe-returned" [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR10]] { -; TUNIT-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR13:[0-9]+]] +; TUNIT-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR14:[0-9]+]] ; TUNIT-NEXT: ret ptr [[R]] ; ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none) ; CGSCC-LABEL: define {{[^@]+}}@aligned_8_return_caller ; CGSCC-SAME: (ptr nofree readnone align 16 [[A:%.*]], i1 noundef [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR13:[0-9]+]] { -; CGSCC-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR14]] +; CGSCC-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR15]] ; CGSCC-NEXT: ret ptr [[R]] ; %r = call ptr @aligned_8_return(ptr %a, i1 %c1, i1 %c2) @@ -1101,6 +1114,104 @@ entry: ret i32 0 } +define i64 @infer_align_atomicrmw(ptr align 4 %p) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12:[0-9]+]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16 +; TUNIT-NEXT: ret i64 [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14:[0-9]+]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16 +; CGSCC-NEXT: ret i64 [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %ret = atomicrmw add ptr %arrayidx1, i64 4 seq_cst, align 16 + ret i64 %ret +} + +define ptr @infer_align_atomicrmw_ptr(ptr align 4 %p, ptr %val) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16 +; TUNIT-NEXT: ret ptr [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16 +; CGSCC-NEXT: ret ptr [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %ret = atomicrmw xchg ptr %arrayidx1, ptr %val seq_cst, align 16 + ret ptr %ret +} + +define i64 @infer_align_cmpxchg(ptr align 4 %p) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16 +; TUNIT-NEXT: [[RET:%.*]] = extractvalue { i64, i1 } [[CMPX]], 0 +; TUNIT-NEXT: ret i64 [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16 +; CGSCC-NEXT: [[RET:%.*]] = extractvalue { i64, i1 } [[CMPX]], 0 +; CGSCC-NEXT: ret i64 [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %cmpx = cmpxchg ptr %arrayidx1, i64 4, i64 1 seq_cst seq_cst, align 16 + %ret = extractvalue { i64, i1 } %cmpx, 0 + ret i64 %ret +} + +define ptr @infer_align_cmpxchg_ptr(ptr align 4 %p, ptr %cmp0, ptr %cmp1) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16 +; TUNIT-NEXT: [[RET:%.*]] = extractvalue { ptr, i1 } [[CMPX]], 0 +; TUNIT-NEXT: ret ptr [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16 +; CGSCC-NEXT: [[RET:%.*]] = extractvalue { ptr, i1 } [[CMPX]], 0 +; CGSCC-NEXT: ret ptr [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %cmpx = cmpxchg ptr %arrayidx1, ptr %cmp0, ptr %cmp1 seq_cst seq_cst, align 16 + %ret = extractvalue { ptr, i1 } %cmpx, 0 + ret ptr %ret +} + declare void @implicit_cast_callee(i64) attributes #0 = { nounwind uwtable noinline } @@ -1119,8 +1230,9 @@ attributes #2 = { null_pointer_is_valid } ; TUNIT: attributes #[[ATTR9]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(write) } ; TUNIT: attributes #[[ATTR10]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) } ; TUNIT: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(read) } -; TUNIT: attributes #[[ATTR12]] = { nofree nosync nounwind willreturn memory(read) } -; TUNIT: attributes #[[ATTR13]] = { nofree nosync nounwind willreturn } +; TUNIT: attributes #[[ATTR12]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) } +; TUNIT: attributes #[[ATTR13]] = { nofree nosync nounwind willreturn memory(read) } +; TUNIT: attributes #[[ATTR14]] = { nofree nosync nounwind willreturn } ;. ; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable } ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable } @@ -1136,6 +1248,7 @@ attributes #2 = { null_pointer_is_valid } ; CGSCC: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) } ; CGSCC: attributes #[[ATTR12]] = { mustprogress nofree nosync nounwind willreturn memory(read) } ; CGSCC: attributes #[[ATTR13]] = { mustprogress nofree nosync nounwind willreturn memory(none) } -; CGSCC: attributes #[[ATTR14]] = { nofree nosync willreturn } -; CGSCC: attributes #[[ATTR15]] = { nofree willreturn memory(read) } +; CGSCC: attributes #[[ATTR14]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) } +; CGSCC: attributes #[[ATTR15]] = { nofree nosync willreturn } +; CGSCC: attributes #[[ATTR16]] = { nofree willreturn memory(read) } ;. diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll index 7d2f0a1351a4..f61388f71c46 100644 --- a/llvm/test/Transforms/Attributor/nocapture-1.ll +++ b/llvm/test/Transforms/Attributor/nocapture-1.ll @@ -524,13 +524,13 @@ define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) { define void @test_cmpxchg(ptr %p) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] { ; TUNIT-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] { ; CGSCC-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4 ; CGSCC-NEXT: ret void ; @@ -541,13 +541,13 @@ define void @test_cmpxchg(ptr %p) { define void @test_cmpxchg_ptr(ptr %p, ptr %q) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg_ptr -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] { ; TUNIT-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg_ptr -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] { ; CGSCC-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8 ; CGSCC-NEXT: ret void ; @@ -558,13 +558,13 @@ define void @test_cmpxchg_ptr(ptr %p, ptr %q) { define void @test_atomicrmw(ptr %p) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_atomicrmw -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR8]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8]] { ; TUNIT-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_atomicrmw -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR11]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11]] { ; CGSCC-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4 ; CGSCC-NEXT: ret void ; diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll index 442464cde389..4df647cf3bb5 100644 --- a/llvm/test/Transforms/Attributor/nofpclass.ll +++ b/llvm/test/Transforms/Attributor/nofpclass.ll @@ -1813,7 +1813,7 @@ define double @fpext(float nofpclass(inf nan) %arg) { define float @atomicrmw_fadd(ptr %ptr, float nofpclass(inf nan) %val) { ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CHECK-LABEL: define float @atomicrmw_fadd -; CHECK-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] { +; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] { ; CHECK-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VAL]] seq_cst, align 4 ; CHECK-NEXT: ret float [[RESULT]] ; -- GitLab From df9ed9cf52f82aed023adc968ca2a0e7f7cccc69 Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:25:07 -0500 Subject: [PATCH 005/933] [mlir][transform] Fix failure in flattening already flattened linalg ops (#86037) The previous implementation was doing an early successful return on `rank <= 1` without adding the original op to transform results. This resulted in errors about number of returns. This patch fixes this by adding the original op to results. Additionally, we first check if op is elementwise and return a slienceable failure early if not. --- .../TransformOps/LinalgTransformOps.cpp | 15 ++++++++----- .../Dialect/Linalg/flatten-elementwise.mlir | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index d82a6beb1086..ecf998312482 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -3269,15 +3269,20 @@ DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::ApplyToEachResultList &results, transform::TransformState &state) { rewriter.setInsertionPoint(target); - if (target.getNumLoops() <= 1) + if (!isElementwise(target)) { + failed(rewriter.notifyMatchFailure( + target, "only elementwise flattening is supported")); + return emitDefaultSilenceableFailure(target); + } + // If rank <= 1, do nothing + if (target.getNumLoops() <= 1) { + results.push_back(target); return DiagnosedSilenceableFailure::success(); + } ReassociationIndices reassociation(target.getNumLoops()); std::iota(reassociation.begin(), reassociation.end(), 0); auto maybeFlattened = - (isElementwise(target)) - ? collapseOpIterationDims(target, reassociation, rewriter) - : FailureOr(rewriter.notifyMatchFailure( - target, "only elementwise flattening is supported")); + collapseOpIterationDims(target, reassociation, rewriter); if (failed(maybeFlattened)) return emitDefaultSilenceableFailure(target); results.push_back(maybeFlattened->collapsedOp); diff --git a/mlir/test/Dialect/Linalg/flatten-elementwise.mlir b/mlir/test/Dialect/Linalg/flatten-elementwise.mlir index 858c133dd536..5a27fe76b134 100644 --- a/mlir/test/Dialect/Linalg/flatten-elementwise.mlir +++ b/mlir/test/Dialect/Linalg/flatten-elementwise.mlir @@ -67,6 +67,27 @@ module attributes {transform.with_named_sequence} { // ----- +// CHECK-LABEL: func.func @map_already_flat( +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-SAME: %[[ARG2:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-NEXT: linalg.map { arith.addf } ins(%[[ARG0]], %[[ARG1]] : memref<32xf32>, memref<32xf32>) outs(%[[ARG2]] : memref<32xf32>) +func.func @map_already_flat(%arg0: memref<32xf32>, %arg1: memref<32xf32>, %arg2: memref<32xf32>) { + linalg.map {arith.addf} ins(%arg0, %arg1: memref<32xf32>, memref<32xf32>) outs(%arg2: memref<32xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} + +// ----- + // CHECK: #[[$MAP0:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func.func @generic // CHECK-SAME: %[[ARG0:[a-zA-Z0-9_]*]]: memref<32x7xf32> -- GitLab From 26c290b46ac6b4a81feb28ae1862fac961138a24 Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 01:42:59 -0400 Subject: [PATCH 006/933] [cmake] Place clang behind mlir in the list of external projects (#86050) In preparation for the initial ClangIR upstreaming process, move clang behind MLIR in the list of external projects. Otherwise, cmake will attempt to build clang before MLIR. --- llvm/tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index c6116ac81d12..2969877b9ee7 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,11 +37,11 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. -add_llvm_external_project(clang) add_llvm_external_project(lld) add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# Flang depends on mlir, so place it afterward +# ClangIR and Flang depends on mlir, so place them afterwards +add_llvm_external_project(clang) add_llvm_external_project(flang) add_llvm_external_project(bolt) -- GitLab From d59730d7060f33dd1607be1fd7813be78759a953 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 20 Mar 2024 22:45:38 -0700 Subject: [PATCH 007/933] [CMake] Change GCC_INSTALL_PREFIX from warning to fatal error (#85891) unless USE_DEPRECATED_GCC_INSTALL_PREFIX (temporary escape hatch) is set. Setting GCC_INSTALL_PREFIX leads to a warning for Clang 18.1 (#77537) and will be completely removed for Clang 20. Link: discourse.llvm.org/t/add-gcc-install-dir-deprecate-gcc-toolchain-and-remove-gcc-install-prefix/65091 Link: discourse.llvm.org/t/correct-cmake-parameters-for-building-clang-and-lld-for-riscv/72833 --- clang/CMakeLists.txt | 5 +++-- clang/docs/ReleaseNotes.rst | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 47fc2e4886cf..761dab8c28c1 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -190,11 +190,12 @@ set(CLANG_RESOURCE_DIR "" CACHE STRING set(C_INCLUDE_DIRS "" CACHE STRING "Colon separated list of directories clang will search for headers.") +set(USE_DEPRECATED_GCC_INSTALL_PREFIX OFF CACHE BOOL "Temporary workaround before GCC_INSTALL_PREFIX is completely removed") set(GCC_INSTALL_PREFIX "" CACHE PATH "Directory where gcc is installed." ) set(DEFAULT_SYSROOT "" CACHE STRING "Default to all compiler invocations for --sysroot=." ) -if(GCC_INSTALL_PREFIX) - message(WARNING "GCC_INSTALL_PREFIX is deprecated and will be removed. Use " +if(GCC_INSTALL_PREFIX AND NOT USE_DEPRECATED_GCC_INSTALL_PREFIX) + message(FATAL_ERROR "GCC_INSTALL_PREFIX is deprecated and will be removed. Use " "configuration files (https://clang.llvm.org/docs/UsersManual.html#configuration-files)" "to specify the default --gcc-install-dir= or --gcc-triple=. --gcc-toolchain= is discouraged. " "See https://github.com/llvm/llvm-project/pull/77537 for detail.") diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c0b0c8a8a3ea..a9c55ef662a0 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -37,6 +37,9 @@ These changes are ones which we think may surprise users when upgrading to Clang |release| because of the opportunity they pose for disruption to existing code bases. +- Setting the deprecated CMake variable ``GCC_INSTALL_PREFIX`` (which sets the + default ``--gcc-toolchain=``) now leads to a fatal error. + C/C++ Language Potentially Breaking Changes ------------------------------------------- -- GitLab From ce8e86971036cb34c3d32cf0b70169379c85ae2f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 22:53:51 -0700 Subject: [PATCH 008/933] [RISCV] Convert an assertion to an if condition in getRegAllocationHints (#85998) With GPR pairs from Zdinx, we can't guarantee there are no subregisters on integer instruction operands. I've been able to get these assertions to fire after some other recent PRs. I've added a FIXME to support this properly. I just wanted to prevent the assertion failure for now. No test case because my other patch #85982 that allowed me to fail the assert hasn't been approved yet, and I don't know for that that patch is required to hit this assert. It's just what exposed it for me. So I think this patch is a good precaution regardless. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 10bf1e88d741..952d17468da5 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -741,8 +741,11 @@ bool RISCVRegisterInfo::getRegAllocationHints( bool NeedGPRC) -> void { Register Reg = MO.getReg(); Register PhysReg = Reg.isPhysical() ? Reg : Register(VRM->getPhys(Reg)); - if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg))) { - assert(!MO.getSubReg() && !VRRegMO.getSubReg() && "Unexpected subreg!"); + // TODO: Support GPRPair subregisters? Need to be careful with even/odd + // registers. If the virtual register is an odd register of a pair and the + // physical register is even (or vice versa), we should not add the hint. + if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg)) && + !MO.getSubReg() && !VRRegMO.getSubReg()) { if (!MRI->isReserved(PhysReg) && !is_contained(Hints, PhysReg)) TwoAddrHints.insert(PhysReg); } -- GitLab From cbcdf126ccc774c063b5d5140c1393ff5305dded Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Wed, 20 Mar 2024 23:50:47 -0700 Subject: [PATCH 009/933] Revert "[cmake] Place clang behind mlir in the list of external projects (#86050)" This reverts commit 26c290b46ac6b4a81feb28ae1862fac961138a24. --- llvm/tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index 2969877b9ee7..c6116ac81d12 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,11 +37,11 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. +add_llvm_external_project(clang) add_llvm_external_project(lld) add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# ClangIR and Flang depends on mlir, so place them afterwards -add_llvm_external_project(clang) +# Flang depends on mlir, so place it afterward add_llvm_external_project(flang) add_llvm_external_project(bolt) -- GitLab From 8fb2160a76b5f051f4cc8f5c8c097830bc91c22c Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 21 Mar 2024 15:38:43 +0800 Subject: [PATCH 010/933] [RISCV] Use DenseMap to track V0 definition. NFC (#84465) Reviving some of the progress on #71764. To recap, we explored removing the V0 register copies to simplify the pass, but hit a limitation with the register allocator due to our use of the vmv0 singleton reg class and early-clobber constraints. So since we will have to continue to track the definition of V0 ourselves, this patch simplifies it by storing it in a map. It will allow us to move about copies to V0 in #71764 without having to do extra bookkeeping. --- llvm/lib/Target/RISCV/RISCVFoldMasks.cpp | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp index fddbaa97d063..2089f5dda6fe 100644 --- a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp +++ b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp @@ -47,10 +47,13 @@ public: StringRef getPassName() const override { return "RISC-V Fold Masks"; } private: - bool convertToUnmasked(MachineInstr &MI, MachineInstr *MaskDef) const; - bool convertVMergeToVMv(MachineInstr &MI, MachineInstr *MaskDef) const; + bool convertToUnmasked(MachineInstr &MI) const; + bool convertVMergeToVMv(MachineInstr &MI) const; - bool isAllOnesMask(MachineInstr *MaskDef) const; + bool isAllOnesMask(const MachineInstr *MaskDef) const; + + /// Maps uses of V0 to the corresponding def of V0. + DenseMap V0Defs; }; } // namespace @@ -59,10 +62,9 @@ char RISCVFoldMasks::ID = 0; INITIALIZE_PASS(RISCVFoldMasks, DEBUG_TYPE, "RISC-V Fold Masks", false, false) -bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) const { - if (!MaskDef) - return false; - assert(MaskDef->isCopy() && MaskDef->getOperand(0).getReg() == RISCV::V0); +bool RISCVFoldMasks::isAllOnesMask(const MachineInstr *MaskDef) const { + assert(MaskDef && MaskDef->isCopy() && + MaskDef->getOperand(0).getReg() == RISCV::V0); Register SrcReg = TRI->lookThruCopyLike(MaskDef->getOperand(1).getReg(), MRI); if (!SrcReg.isVirtual()) return false; @@ -89,8 +91,7 @@ bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) const { // Transform (VMERGE_VVM_ false, false, true, allones, vl, sew) to // (VMV_V_V_ false, true, vl, sew). It may decrease uses of VMSET. -bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, - MachineInstr *V0Def) const { +bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI) const { #define CASE_VMERGE_TO_VMV(lmul) \ case RISCV::PseudoVMERGE_VVM_##lmul: \ NewOpc = RISCV::PseudoVMV_V_V_##lmul; \ @@ -116,7 +117,7 @@ bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, return false; assert(MI.getOperand(4).isReg() && MI.getOperand(4).getReg() == RISCV::V0); - if (!isAllOnesMask(V0Def)) + if (!isAllOnesMask(V0Defs.lookup(&MI))) return false; MI.setDesc(TII->get(NewOpc)); @@ -133,14 +134,13 @@ bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, return true; } -bool RISCVFoldMasks::convertToUnmasked(MachineInstr &MI, - MachineInstr *MaskDef) const { +bool RISCVFoldMasks::convertToUnmasked(MachineInstr &MI) const { const RISCV::RISCVMaskedPseudoInfo *I = RISCV::getMaskedPseudoInfo(MI.getOpcode()); if (!I) return false; - if (!isAllOnesMask(MaskDef)) + if (!isAllOnesMask(V0Defs.lookup(&MI))) return false; // There are two classes of pseudos in the table - compares and @@ -198,20 +198,26 @@ bool RISCVFoldMasks::runOnMachineFunction(MachineFunction &MF) { // $v0:vr = COPY %mask:vr // %x:vr = Pseudo_MASK %a:vr, %b:br, $v0:vr // - // Because $v0 isn't in SSA, keep track of it so we can check the mask operand - // on each pseudo. - MachineInstr *CurrentV0Def; - for (MachineBasicBlock &MBB : MF) { - CurrentV0Def = nullptr; - for (MachineInstr &MI : MBB) { - Changed |= convertToUnmasked(MI, CurrentV0Def); - Changed |= convertVMergeToVMv(MI, CurrentV0Def); + // Because $v0 isn't in SSA, keep track of its definition at each use so we + // can check mask operands. + for (const MachineBasicBlock &MBB : MF) { + const MachineInstr *CurrentV0Def = nullptr; + for (const MachineInstr &MI : MBB) { + if (MI.readsRegister(RISCV::V0, TRI)) + V0Defs[&MI] = CurrentV0Def; if (MI.definesRegister(RISCV::V0, TRI)) CurrentV0Def = &MI; } } + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + Changed |= convertToUnmasked(MI); + Changed |= convertVMergeToVMv(MI); + } + } + return Changed; } -- GitLab From adda597388dc148ac235e755b3e8bbd0a12a3e15 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Thu, 21 Mar 2024 09:07:57 +0100 Subject: [PATCH 011/933] [MLIR] Add index bitwidth to the DataLayout (#85927) When importing from LLVM IR the data layout of all pointer types contains an index bitwidth that should be used for index computations. This revision adds a getter to the DataLayout that provides access to the already stored bitwidth. The function returns an optional since only pointer-like types have an index bitwidth. Querying the bitwidth of a non-pointer type returns std::nullopt. The new function works for the built-in Index type and, using a type interface, for the LLVMPointerType. --- mlir/docs/DataLayout.md | 4 +- mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td | 2 +- .../mlir/Interfaces/DataLayoutInterfaces.h | 13 +++++ .../mlir/Interfaces/DataLayoutInterfaces.td | 28 ++++++++++ mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp | 25 +++++++-- mlir/lib/Interfaces/DataLayoutInterfaces.cpp | 30 ++++++++++- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 11 ++-- mlir/test/Dialect/LLVMIR/layout.mlir | 30 +++++++---- .../DataLayoutInterfaces/module.mlir | 4 +- .../DataLayoutInterfaces/query.mlir | 51 ++++++++++++++++--- .../DataLayoutInterfaces/types.mlir | 1 + mlir/test/Target/LLVMIR/data-layout.mlir | 2 +- .../lib/Dialect/DLTI/TestDataLayoutQuery.cpp | 12 +++-- mlir/test/lib/Dialect/Test/TestTypeDefs.td | 3 +- mlir/test/lib/Dialect/Test/TestTypes.cpp | 8 ++- .../Interfaces/DataLayoutInterfacesTest.cpp | 7 +++ 16 files changed, 193 insertions(+), 38 deletions(-) diff --git a/mlir/docs/DataLayout.md b/mlir/docs/DataLayout.md index b9dde30519d6..86ad51a517ae 100644 --- a/mlir/docs/DataLayout.md +++ b/mlir/docs/DataLayout.md @@ -77,6 +77,7 @@ public: llvm::TypeSize getTypeSizeInBits(Type type) const; uint64_t getTypeABIAlignment(Type type) const; uint64_t getTypePreferredAlignment(Type type) const; + std::optional getTypeIndexBitwidth(Type type) const; }; ``` @@ -267,7 +268,8 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< >} {} ``` -specifies that `index` has 32 bits. All other layout properties of `index` match +specifies that `index` has 32 bits and index computations should be performed +using 32-bit precision as well. All other layout properties of `index` match those of the integer type with the same bitwidth defined above. In absence of the corresponding entry, `index` is assumed to be a 64-bit diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td index 96cdbf01b4bd..b7176aa93ff1 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td @@ -123,7 +123,7 @@ def LLVMFunctionType : LLVMType<"LLVMFunction", "func"> { def LLVMPointerType : LLVMType<"LLVMPointer", "ptr", [ DeclareTypeInterfaceMethods]> { + "getIndexBitwidth", "areCompatible", "verifyEntries"]>]> { let summary = "LLVM pointer type"; let description = [{ The `!llvm.ptr` type is an LLVM pointer type. This type typically represents diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h index 4a21f76dfc5d..046354677e6a 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h @@ -57,6 +57,13 @@ uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef params); +/// Default handler for the index bitwidth request. Computes the result for +/// the built-in index type and dispatches to the DataLayoutTypeInterface for +/// other types. +std::optional +getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, + ArrayRef params); + /// Default handler for alloca memory space request. Dispatches to the /// DataLayoutInterface if specified, otherwise returns the default. Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry); @@ -180,6 +187,11 @@ public: /// Returns the preferred of the given type in the current scope. uint64_t getTypePreferredAlignment(Type t) const; + /// Returns the bitwidth that should be used when performing index + /// computations for the given pointer-like type in the current scope. If the + /// type is not a pointer-like type, it returns std::nullopt. + std::optional getTypeIndexBitwidth(Type t) const; + /// Returns the memory space used for AllocaOps. Attribute getAllocaMemorySpace() const; @@ -216,6 +228,7 @@ private: mutable DenseMap bitsizes; mutable DenseMap abiAlignments; mutable DenseMap preferredAlignments; + mutable DenseMap> indexBitwidths; /// Cache for alloca, global, and program memory spaces. mutable std::optional allocaMemorySpace; diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td index a8def967fffc..0ee7a116d114 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td @@ -280,6 +280,22 @@ def DataLayoutOpInterface : OpInterface<"DataLayoutOpInterface"> { params); }] >, + StaticInterfaceMethod< + /*description=*/"Returns the bitwidth that should be used when " + "performing index computations for the type computed " + "using the relevant entries. The data layout object can " + "be used for recursive queries.", + /*retTy=*/"std::optional", + /*methodName=*/"getIndexBitwidth", + /*args=*/(ins "::mlir::Type":$type, + "const ::mlir::DataLayout &":$dataLayout, + "::mlir::DataLayoutEntryListRef":$params), + /*methodBody=*/"", + /*defaultImplementation=*/[{ + return ::mlir::detail::getDefaultIndexBitwidth(type, dataLayout, + params); + }] + >, StaticInterfaceMethod< /*description=*/"Returns the memory space used by the ABI computed " "using the relevant entries. The data layout object " @@ -400,6 +416,18 @@ def DataLayoutTypeInterface : TypeInterface<"DataLayoutTypeInterface"> { /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout, "::mlir::DataLayoutEntryListRef":$params) >, + InterfaceMethod< + /*description=*/"Returns the bitwidth that should be used when " + "performing index computations for the given " + "pointer-like type. If the type is not a pointer-like " + "type, returns std::nullopt.", + /*retTy=*/"std::optional", + /*methodName=*/"getIndexBitwidth", + /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout, + "::mlir::DataLayoutEntryListRef":$params), + /*methodBody=*/"", + /*defaultImplementation=*/[{ return std::nullopt; }] + >, InterfaceMethod< /*desc=*/"Returns true if the two lists of entries are compatible, that " "is, that `newLayout` spec entries can be nested in an op with " diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp index 443e245887ea..630187f220a4 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp @@ -287,15 +287,22 @@ getPointerDataLayoutEntry(DataLayoutEntryListRef params, LLVMPointerType type, } } if (currentEntry) { - return *extractPointerSpecValue(currentEntry, pos) / - (pos == PtrDLEntryPos::Size ? 1 : kBitsInByte); + std::optional value = extractPointerSpecValue(currentEntry, pos); + // If the optional `PtrDLEntryPos::Index` entry is not available, use the + // pointer size as the index bitwidth. + if (!value && pos == PtrDLEntryPos::Index) + value = extractPointerSpecValue(currentEntry, PtrDLEntryPos::Size); + bool isSizeOrIndex = + pos == PtrDLEntryPos::Size || pos == PtrDLEntryPos::Index; + return *value / (isSizeOrIndex ? 1 : kBitsInByte); } // If not found, and this is the pointer to the default memory space, assume // 64-bit pointers. if (type.getAddressSpace() == 0) { - return pos == PtrDLEntryPos::Size ? kDefaultPointerSizeBits - : kDefaultPointerAlignment; + bool isSizeOrIndex = + pos == PtrDLEntryPos::Size || pos == PtrDLEntryPos::Index; + return isSizeOrIndex ? kDefaultPointerSizeBits : kDefaultPointerAlignment; } return std::nullopt; @@ -332,6 +339,16 @@ LLVMPointerType::getPreferredAlignment(const DataLayout &dataLayout, return dataLayout.getTypePreferredAlignment(get(getContext())); } +std::optional +LLVMPointerType::getIndexBitwidth(const DataLayout &dataLayout, + DataLayoutEntryListRef params) const { + if (std::optional indexBitwidth = + getPointerDataLayoutEntry(params, *this, PtrDLEntryPos::Index)) + return *indexBitwidth; + + return dataLayout.getTypeIndexBitwidth(get(getContext())); +} + bool LLVMPointerType::areCompatible(DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout) const { for (DataLayoutEntryInterface newEntry : newLayout) { diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp index 65c41f44192a..b5b7d78cfeff 100644 --- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp +++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp @@ -218,7 +218,23 @@ uint64_t mlir::detail::getDefaultPreferredAlignment( reportMissingDataLayout(type); } -// Returns the memory space used for allocal operations if specified in the +std::optional mlir::detail::getDefaultIndexBitwidth( + Type type, const DataLayout &dataLayout, + ArrayRef params) { + if (isa(type)) + return getIndexBitwidth(params); + + if (auto typeInterface = dyn_cast(type)) + if (std::optional indexBitwidth = + typeInterface.getIndexBitwidth(dataLayout, params)) + return *indexBitwidth; + + // Return std::nullopt for all other types, which are assumed to be non + // pointer-like types. + return std::nullopt; +} + +// Returns the memory space used for alloca operations if specified in the // given entry. If the entry is empty the default memory space represented by // an empty attribute is returned. Attribute @@ -520,6 +536,18 @@ uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const { }); } +std::optional mlir::DataLayout::getTypeIndexBitwidth(Type t) const { + checkValid(); + return cachedLookup>(t, indexBitwidths, [&](Type ty) { + DataLayoutEntryList list; + if (originalLayout) + list = originalLayout.getSpecForType(ty.getTypeID()); + if (auto iface = dyn_cast_or_null(scope)) + return iface.getIndexBitwidth(ty, *this, list); + return detail::getDefaultIndexBitwidth(ty, *this, list); + }); +} + mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const { checkValid(); if (allocaMemorySpace) diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 995544238e4a..f90495d407fd 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -274,16 +274,15 @@ translateDataLayout(DataLayoutSpecInterface attribute, layoutStream << ":" << preferred; return success(); }) - .Case([&](LLVMPointerType ptrType) { - layoutStream << "p" << ptrType.getAddressSpace() << ":"; + .Case([&](LLVMPointerType type) { + layoutStream << "p" << type.getAddressSpace() << ":"; uint64_t size = dataLayout.getTypeSizeInBits(type); uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u; uint64_t preferred = dataLayout.getTypePreferredAlignment(type) * 8u; - layoutStream << size << ":" << abi << ":" << preferred; - if (std::optional index = extractPointerSpecValue( - entry.getValue(), PtrDLEntryPos::Index)) - layoutStream << ":" << *index; + uint64_t index = *dataLayout.getTypeIndexBitwidth(type); + layoutStream << size << ":" << abi << ":" << preferred << ":" + << index; return success(); }) .Default([loc](Type type) { diff --git a/mlir/test/Dialect/LLVMIR/layout.mlir b/mlir/test/Dialect/LLVMIR/layout.mlir index 2868e1740f86..a78fb771242e 100644 --- a/mlir/test/Dialect/LLVMIR/layout.mlir +++ b/mlir/test/Dialect/LLVMIR/layout.mlir @@ -7,6 +7,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -16,6 +17,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -25,6 +27,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -39,7 +42,7 @@ module { module attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry : vector<3xi64>>, #dlti.dl_entry, dense<[64, 64, 64]> : vector<3xi64>>, - #dlti.dl_entry, dense<[32, 64, 64]> : vector<3xi64>>, + #dlti.dl_entry, dense<[32, 64, 64, 24]> : vector<4xi64>>, #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>, #dlti.dl_entry<"dlti.global_memory_space", 2 : ui64>, #dlti.dl_entry<"dlti.program_memory_space", 3 : ui64>, @@ -51,6 +54,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 32 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -60,6 +64,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 32 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -69,24 +74,17 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 64 // CHECK: global_memory_space = 2 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 8 // CHECK: stack_alignment = 128 "test.data_layout_query"() : () -> !llvm.ptr<5> - // CHECK: alignment = 4 - // CHECK: alloca_memory_space = 5 - // CHECK: bitsize = 32 - // CHECK: global_memory_space = 2 - // CHECK: preferred = 8 - // CHECK: program_memory_space = 3 - // CHECK: size = 4 - // CHECK: stack_alignment = 128 - "test.data_layout_query"() : () -> !llvm.ptr<3> // CHECK: alignment = 8 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 24 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -134,6 +132,7 @@ module { // simple case // CHECK: alignment = 4 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> !llvm.struct<(i32)> @@ -141,6 +140,7 @@ module { // padding inbetween // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(i32, f64)> @@ -148,6 +148,7 @@ module { // padding at end of struct // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(f64, i32)> @@ -155,6 +156,7 @@ module { // packed // CHECK: alignment = 1 // CHECK: bitsize = 96 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 12 "test.data_layout_query"() : () -> !llvm.struct @@ -162,6 +164,7 @@ module { // empty // CHECK: alignment = 1 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 1 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.struct<()> @@ -179,6 +182,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // Strict alignment is applied // CHECK: alignment = 4 // CHECK: bitsize = 16 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 2 "test.data_layout_query"() : () -> !llvm.struct<(i16)> @@ -186,6 +190,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // No impact on structs that have stricter requirements // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(i32, f64)> @@ -193,6 +198,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // Only the preferred alignment of structs is affected // CHECK: alignment = 1 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> !llvm.struct @@ -200,6 +206,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // empty // CHECK: alignment = 4 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.struct<()> @@ -265,6 +272,7 @@ module { // simple case // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 8 "test.data_layout_query"() : () -> !llvm.array<2 x i32> @@ -272,6 +280,7 @@ module { // size 0 // CHECK: alignment = 8 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.array<0 x f64> @@ -279,6 +288,7 @@ module { // alignment info matches element type // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 8 "test.data_layout_query"() : () -> !llvm.array<1 x i64> diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir index 096e7ceb3cbc..97286ce75806 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir @@ -2,11 +2,13 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 12]>, - #dlti.dl_entry, ["alignment", 32]>>} { + #dlti.dl_entry, ["alignment", 32]>, + #dlti.dl_entry, ["index", 7]>>} { // CHECK-LABEL: @module_level_layout func.func @module_level_layout() { // CHECK: alignment = 32 // CHECK: bitsize = 12 + // CHECK: index = 7 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir index 9f9240ac6f8c..d3bc91339d16 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir @@ -4,24 +4,34 @@ func.func @no_layout_builtin() { // CHECK: alignment = 4 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 8 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 8 "test.data_layout_query"() : () -> f64 // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 8 "test.data_layout_query"() : () -> complex // CHECK: alignment = 1 // CHECK: bitsize = 14 + // CHECK: index = 0 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> complex + // CHECK: alignment = 4 + // CHECK: bitsize = 64 + // CHECK: index = 64 + // CHECK: preferred = 8 + // CHECK: size = 8 + "test.data_layout_query"() : () -> index return } @@ -30,6 +40,7 @@ func.func @no_layout_builtin() { func.func @no_layout_custom() { // CHECK: alignment = 1 // CHECK: bitsize = 1 + // CHECK: index = 1 // CHECK: preferred = 1 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -41,6 +52,7 @@ func.func @layout_op_no_layout() { "test.op_with_data_layout"() ({ // CHECK: alignment = 1 // CHECK: bitsize = 1 + // CHECK: index = 1 // CHECK: preferred = 1 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<1000> @@ -54,13 +66,15 @@ func.func @layout_op() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () return } @@ -72,13 +86,15 @@ func.func @nested_inner_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () "test.maybe_terminator"() : () -> () }) : () -> () @@ -92,6 +108,7 @@ func.func @nested_outer_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -100,7 +117,8 @@ func.func @nested_outer_only() { "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () return } @@ -112,6 +130,7 @@ func.func @nested_middle_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -120,7 +139,8 @@ func.func @nested_middle_only() { "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () "test.maybe_terminator"() : () -> () }) : () -> () @@ -134,6 +154,7 @@ func.func @nested_combine_with_missing() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 21 // CHECK: preferred = 30 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -146,13 +167,15 @@ func.func @nested_combine_with_missing() { >} : () -> () // CHECK: alignment = 1 // CHECK: bitsize = 42 + // CHECK: index = 21 // CHECK: preferred = 30 // CHECK: size = 6 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 42]>, - #dlti.dl_entry, ["preferred", 30]> + #dlti.dl_entry, ["preferred", 30]>, + #dlti.dl_entry, ["index", 21]> >}: () -> () return } @@ -164,6 +187,7 @@ func.func @nested_combine_all() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 3 + // CHECK: index = 40 // CHECK: preferred = 30 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -174,16 +198,19 @@ func.func @nested_combine_all() { >} : () -> () // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 40 // CHECK: preferred = 30 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 40]> >} : () -> () // CHECK: alignment = 1 // CHECK: bitsize = 42 + // CHECK: index = 1 // CHECK: preferred = 30 // CHECK: size = 6 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -200,18 +227,22 @@ func.func @integers() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 8 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 16 // CHECK: bitsize = 56 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i56 // CHECK: alignment = 16 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i64 // CHECK: alignment = 16 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i128 "test.maybe_terminator"() : () -> () @@ -222,18 +253,22 @@ func.func @integers() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 16 // CHECK: bitsize = 56 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i56 // CHECK: alignment = 16 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i64 // CHECK: alignment = 16 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i128 "test.maybe_terminator"() : () -> () @@ -248,10 +283,12 @@ func.func @floats() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 8 "test.data_layout_query"() : () -> f32 // CHECK: alignment = 16 // CHECK: bitsize = 80 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> f80 "test.maybe_terminator"() : () -> () @@ -262,10 +299,12 @@ func.func @floats() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> f32 // CHECK: alignment = 16 // CHECK: bitsize = 80 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> f80 "test.maybe_terminator"() : () -> () diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir index 55bb1d2eac91..82ae02cf92ad 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir @@ -40,6 +40,7 @@ module @index attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry>} { func.func @query() { // CHECK: bitsize = 32 + // CHECK: index = 32 "test.data_layout_query"() : () -> index return } diff --git a/mlir/test/Target/LLVMIR/data-layout.mlir b/mlir/test/Target/LLVMIR/data-layout.mlir index e61972a0dd97..881d6727e2a1 100644 --- a/mlir/test/Target/LLVMIR/data-layout.mlir +++ b/mlir/test/Target/LLVMIR/data-layout.mlir @@ -6,7 +6,7 @@ // CHECK: S128- // CHECK: i64:64:128 // CHECK: f80:128:256 -// CHECK: p0:32:64:128 +// CHECK: p0:32:64:128:32 // CHECK: p1:32:32:32:16 module attributes {dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry<"dlti.endianness", "big">, diff --git a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp index 740562e77830..3da48ffa403e 100644 --- a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp +++ b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp @@ -36,19 +36,21 @@ struct TestDataLayoutQuery return; const DataLayout &layout = layouts.getAbove(op); - unsigned size = layout.getTypeSize(op.getType()); - unsigned bitsize = layout.getTypeSizeInBits(op.getType()); - unsigned alignment = layout.getTypeABIAlignment(op.getType()); - unsigned preferred = layout.getTypePreferredAlignment(op.getType()); + llvm::TypeSize size = layout.getTypeSize(op.getType()); + llvm::TypeSize bitsize = layout.getTypeSizeInBits(op.getType()); + uint64_t alignment = layout.getTypeABIAlignment(op.getType()); + uint64_t preferred = layout.getTypePreferredAlignment(op.getType()); + uint64_t index = layout.getTypeIndexBitwidth(op.getType()).value_or(0); Attribute allocaMemorySpace = layout.getAllocaMemorySpace(); Attribute programMemorySpace = layout.getProgramMemorySpace(); Attribute globalMemorySpace = layout.getGlobalMemorySpace(); - unsigned stackAlignment = layout.getStackAlignment(); + uint64_t stackAlignment = layout.getStackAlignment(); op->setAttrs( {builder.getNamedAttr("size", builder.getIndexAttr(size)), builder.getNamedAttr("bitsize", builder.getIndexAttr(bitsize)), builder.getNamedAttr("alignment", builder.getIndexAttr(alignment)), builder.getNamedAttr("preferred", builder.getIndexAttr(preferred)), + builder.getNamedAttr("index", builder.getIndexAttr(index)), builder.getNamedAttr("alloca_memory_space", allocaMemorySpace == Attribute() ? builder.getUI32IntegerAttr(0) diff --git a/mlir/test/lib/Dialect/Test/TestTypeDefs.td b/mlir/test/lib/Dialect/Test/TestTypeDefs.td index 1957845c842f..492642b711e0 100644 --- a/mlir/test/lib/Dialect/Test/TestTypeDefs.td +++ b/mlir/test/lib/Dialect/Test/TestTypeDefs.td @@ -148,7 +148,8 @@ def TestType : Test_Type<"Test", [ } def TestTypeWithLayoutType : Test_Type<"TestTypeWithLayout", [ - DeclareTypeInterfaceMethods + DeclareTypeInterfaceMethods ]> { let mnemonic = "test_type_with_layout"; let parameters = (ins "unsigned":$key); diff --git a/mlir/test/lib/Dialect/Test/TestTypes.cpp b/mlir/test/lib/Dialect/Test/TestTypes.cpp index 2f4c9b689069..7a195eb25a3b 100644 --- a/mlir/test/lib/Dialect/Test/TestTypes.cpp +++ b/mlir/test/lib/Dialect/Test/TestTypes.cpp @@ -276,6 +276,12 @@ uint64_t TestTypeWithLayoutType::getPreferredAlignment( return extractKind(params, "preferred"); } +std::optional +TestTypeWithLayoutType::getIndexBitwidth(const DataLayout &dataLayout, + DataLayoutEntryListRef params) const { + return extractKind(params, "index"); +} + bool TestTypeWithLayoutType::areCompatible( DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout) const { unsigned old = extractKind(oldLayout, "alignment"); @@ -297,7 +303,7 @@ TestTypeWithLayoutType::verifyEntries(DataLayoutEntryListRef params, (void)kind; assert(kind && (kind.getValue() == "size" || kind.getValue() == "alignment" || - kind.getValue() == "preferred") && + kind.getValue() == "preferred" || kind.getValue() == "index") && "unexpected kind"); assert(llvm::isa(array.getValue().back())); } diff --git a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp index 794e19710fad..d6b8d7392f32 100644 --- a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp +++ b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp @@ -345,6 +345,8 @@ TEST(DataLayout, NullSpec) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 32u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); @@ -373,6 +375,8 @@ TEST(DataLayout, EmptySpec) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 32u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); @@ -385,6 +389,7 @@ TEST(DataLayout, SpecWithEntries) { "dltest.op_with_layout"() { dltest.layout = #dltest.spec< #dlti.dl_entry, #dlti.dl_entry, + #dlti.dl_entry, #dlti.dl_entry<"dltest.alloca_memory_space", 5 : i32>, #dlti.dl_entry<"dltest.program_memory_space", 3 : i32>, #dlti.dl_entry<"dltest.global_memory_space", 2 : i32>, @@ -408,6 +413,8 @@ TEST(DataLayout, SpecWithEntries) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 8u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 16u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 42u); EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 32)), 32u); EXPECT_EQ(layout.getTypeSize(Float32Type::get(&ctx)), 32u); -- GitLab From a29e9e32c50273abffc53e3700bbc23985f0a7af Mon Sep 17 00:00:00 2001 From: Roberto Bampi Date: Thu, 21 Mar 2024 09:29:25 +0100 Subject: [PATCH 012/933] [clang-format] Add --fail-on-incomplete-format. (#84346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment clang-format will return exit code 0 on incomplete results. In scripts it would sometimes be useful if clang-format would instead fail in those cases, signalling that there was something wrong with the code being formatted. --------- Co-authored-by: Björn Schäpers Co-authored-by: Owen Pan --- clang/docs/ClangFormat.rst | 1 + clang/test/Format/fail-on-incomplete.cpp | 4 ++++ clang/tools/clang-format/ClangFormat.cpp | 13 +++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 clang/test/Format/fail-on-incomplete.cpp diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst index 819d9ee9f9cd..80dc38a075c8 100644 --- a/clang/docs/ClangFormat.rst +++ b/clang/docs/ClangFormat.rst @@ -61,6 +61,7 @@ to format C/C++/Java/JavaScript/JSON/Objective-C/Protobuf/C# code. --dry-run - If set, do not actually make the formatting changes --dump-config - Dump configuration options to stdout and exit. Can be used with -style option. + --fail-on-incomplete-format - If set, fail with exit code 1 on incomplete format. --fallback-style= - The name of the predefined style used as a fallback in case clang-format is invoked with -style=file, but can not find the .clang-format diff --git a/clang/test/Format/fail-on-incomplete.cpp b/clang/test/Format/fail-on-incomplete.cpp new file mode 100644 index 000000000000..ccd77af4d599 --- /dev/null +++ b/clang/test/Format/fail-on-incomplete.cpp @@ -0,0 +1,4 @@ +// RUN: not clang-format -style=LLVM -fail-on-incomplete-format %s +// RUN: clang-format -style=LLVM %s + +int a( diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index e122cea50f72..ed401135ad84 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -205,6 +205,11 @@ static cl::list FileNames(cl::Positional, cl::desc("[@] [ ...]"), cl::cat(ClangFormatCategory)); +static cl::opt FailOnIncompleteFormat( + "fail-on-incomplete-format", + cl::desc("If set, fail with exit code 1 on incomplete format."), + cl::init(false), cl::cat(ClangFormatCategory)); + namespace clang { namespace format { @@ -399,7 +404,7 @@ class ClangFormatDiagConsumer : public DiagnosticConsumer { }; // Returns true on error. -static bool format(StringRef FileName) { +static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) { const bool IsSTDIN = FileName == "-"; if (!OutputXML && Inplace && IsSTDIN) { errs() << "error: cannot use -i when reading from stdin.\n"; @@ -535,7 +540,7 @@ static bool format(StringRef FileName) { Rewrite.getEditBuffer(ID).write(outs()); } } - return false; + return ErrorOnIncompleteFormat && !Status.FormatComplete; } } // namespace format @@ -699,7 +704,7 @@ int main(int argc, const char **argv) { } if (FileNames.empty()) - return clang::format::format("-"); + return clang::format::format("-", FailOnIncompleteFormat); if (FileNames.size() > 1 && (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) { @@ -717,7 +722,7 @@ int main(int argc, const char **argv) { errs() << "Formatting [" << FileNo++ << "/" << FileNames.size() << "] " << FileName << "\n"; } - Error |= clang::format::format(FileName); + Error |= clang::format::format(FileName, FailOnIncompleteFormat); } return Error ? 1 : 0; } -- GitLab From 1404640533fdeda5fb381019a436e4ee84bb2174 Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Thu, 21 Mar 2024 08:32:05 +0000 Subject: [PATCH 013/933] [mlir][Bazel] Add target for index dialect python bindings --- .../mlir/python/BUILD.bazel | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel index 0c3ed22e7360..d6b0832f4c1a 100644 --- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel @@ -492,6 +492,44 @@ filegroup( ], ) +##---------------------------------------------------------------------------## +# Index dialect. +##---------------------------------------------------------------------------## + +gentbl_filegroup( + name = "IndexOpsPyGen", + tbl_outs = [ + ( + [ + "-gen-python-enum-bindings", + "-bind-dialect=index", + ], + "mlir/dialects/_index_enum_gen.py", + ), + ( + [ + "-gen-python-op-bindings", + "-bind-dialect=index", + ], + "mlir/dialects/_index_ops_gen.py", + ), + ], + tblgen = "//mlir:mlir-tblgen", + td_file = "mlir/dialects/IndexOps.td", + deps = [ + "//mlir:IndexOpsTdFiles", + "//mlir:OpBaseTdFiles", + ], +) + +filegroup( + name = "IndexOpsPyFiles", + srcs = [ + "mlir/dialects/index.py", + ":IndexOpsPyGen", + ], +) + ##---------------------------------------------------------------------------## # Math dialect. ##---------------------------------------------------------------------------## -- GitLab From b6b703b2dfc1d1ba45ebc64ed6b53a3a46f531f5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 14:24:06 +0530 Subject: [PATCH 014/933] AMDGPU: Infer no-agpr usage in AMDGPUAttributor (#85948) SIMachineFunctionInfo has a scan of the function body for inline asm which may use AGPRs, or callees in SIMachineFunctionInfo. Move this into the attributor, so it actually works interprocedurally. Could probably avoid most of the test churn if this bothered to avoid adding this on subtargets without AGPRs. We should also probably try to delete the MIR scan in usesAGPRs but it seems to be trickier to eliminate. --- llvm/docs/AMDGPUUsage.rst | 5 + llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 96 ++++++- .../Target/AMDGPU/SIMachineFunctionInfo.cpp | 30 +-- .../AMDGPU/addrspacecast-constantexpr.ll | 6 +- .../AMDGPU/amdgpu-attributor-no-agpr.ll | 255 ++++++++++++++++++ .../annotate-kernel-features-hsa-call.ll | 44 +-- .../AMDGPU/annotate-kernel-features-hsa.ll | 26 +- .../AMDGPU/annotate-kernel-features.ll | 19 +- .../AMDGPU/copy-vgpr-clobber-spill-vgpr.mir | 2 +- .../CodeGen/AMDGPU/direct-indirect-call.ll | 2 +- .../AMDGPU/duplicate-attribute-indirect.ll | 2 +- .../AMDGPU/implicitarg-offset-attributes.ll | 30 +-- .../AMDGPU/preload-kernargs-inreg-hints.ll | 20 +- .../AMDGPU/propagate-flat-work-group-size.ll | 18 +- .../CodeGen/AMDGPU/propagate-waves-per-eu.ll | 44 +-- .../AMDGPU/recursive_global_initializer.ll | 2 +- .../AMDGPU/remove-no-kernel-id-attribute.ll | 8 +- .../CodeGen/AMDGPU/simple-indirect-call.ll | 2 +- .../uniform-work-group-attribute-missing.ll | 4 +- .../AMDGPU/uniform-work-group-multistep.ll | 4 +- ...niform-work-group-nested-function-calls.ll | 4 +- ...ork-group-prevent-attribute-propagation.ll | 6 +- .../uniform-work-group-propagate-attribute.ll | 4 +- .../uniform-work-group-recursion-test.ll | 6 +- .../CodeGen/AMDGPU/uniform-work-group-test.ll | 4 +- .../CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll | 1 + 26 files changed, 485 insertions(+), 159 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 29ea5005c0c4..6e6d6b157148 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1454,6 +1454,11 @@ The AMDGPU backend supports the following LLVM IR attributes. CLANG attribute [CLANG-ATTR]_. Clang only emits this attribute when all the three numbers are >= 1. + "amdgpu-no-agpr" Indicates the function will not require allocating AGPRs. This is only + relevant on subtargets with AGPRs. The behavior is undefined if a + function which requires AGPRs is reached through any function marked + with this attribute. + ======================================= ========================================================== Calling Conventions diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp index d7f5110427ec..9bd30458bc0a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -918,6 +918,96 @@ AAAMDWavesPerEU &AAAMDWavesPerEU::createForPosition(const IRPosition &IRP, llvm_unreachable("AAAMDWavesPerEU is only valid for function position"); } +static bool inlineAsmUsesAGPRs(const InlineAsm *IA) { + for (const auto &CI : IA->ParseConstraints()) { + for (StringRef Code : CI.Codes) { + Code.consume_front("{"); + if (Code.starts_with("a")) + return true; + } + } + + return false; +} + +struct AAAMDGPUNoAGPR + : public IRAttribute, + AAAMDGPUNoAGPR> { + AAAMDGPUNoAGPR(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {} + + static AAAMDGPUNoAGPR &createForPosition(const IRPosition &IRP, + Attributor &A) { + if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION) + return *new (A.Allocator) AAAMDGPUNoAGPR(IRP, A); + llvm_unreachable("AAAMDGPUNoAGPR is only valid for function position"); + } + + void initialize(Attributor &A) override { + Function *F = getAssociatedFunction(); + if (F->hasFnAttribute("amdgpu-no-agpr")) + indicateOptimisticFixpoint(); + } + + const std::string getAsStr(Attributor *A) const override { + return getAssumed() ? "amdgpu-no-agpr" : "amdgpu-maybe-agpr"; + } + + void trackStatistics() const override {} + + ChangeStatus updateImpl(Attributor &A) override { + // TODO: Use AACallEdges, but then we need a way to inspect asm edges. + + auto CheckForNoAGPRs = [&](Instruction &I) { + const auto &CB = cast(I); + const Value *CalleeOp = CB.getCalledOperand(); + const Function *Callee = dyn_cast(CalleeOp); + if (!Callee) { + if (const InlineAsm *IA = dyn_cast(CalleeOp)) + return !inlineAsmUsesAGPRs(IA); + return false; + } + + // Some intrinsics may use AGPRs, but if we have a choice, we are not + // required to use AGPRs. + if (Callee->isIntrinsic()) + return true; + + // TODO: Handle callsite attributes + const auto *CalleeInfo = A.getAAFor( + *this, IRPosition::function(*Callee), DepClassTy::REQUIRED); + return CalleeInfo && CalleeInfo->getAssumed(); + }; + + bool UsedAssumedInformation = false; + if (!A.checkForAllCallLikeInstructions(CheckForNoAGPRs, *this, + UsedAssumedInformation)) + return indicatePessimisticFixpoint(); + return ChangeStatus::UNCHANGED; + } + + ChangeStatus manifest(Attributor &A) override { + if (!getAssumed()) + return ChangeStatus::UNCHANGED; + LLVMContext &Ctx = getAssociatedFunction()->getContext(); + return A.manifestAttrs(getIRPosition(), + {Attribute::get(Ctx, "amdgpu-no-agpr")}); + } + + const std::string getName() const override { return "AAAMDGPUNoAGPR"; } + const char *getIdAddr() const override { return &ID; } + + /// This function should return true if the type of the \p AA is + /// AAAMDGPUNoAGPRs + static bool classof(const AbstractAttribute *AA) { + return (AA->getIdAddr() == &ID); + } + + static const char ID; +}; + +const char AAAMDGPUNoAGPR::ID = 0; + static void addPreloadKernArgHint(Function &F, TargetMachine &TM) { const GCNSubtarget &ST = TM.getSubtarget(F); for (unsigned I = 0; @@ -946,8 +1036,9 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { DenseSet Allowed( {&AAAMDAttributes::ID, &AAUniformWorkGroupSize::ID, &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID, - &AAAMDWavesPerEU::ID, &AACallEdges::ID, &AAPointerInfo::ID, - &AAPotentialConstantValues::ID, &AAUnderlyingObjects::ID}); + &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID, &AACallEdges::ID, + &AAPointerInfo::ID, &AAPotentialConstantValues::ID, + &AAUnderlyingObjects::ID}); AttributorConfig AC(CGUpdater); AC.Allowed = &Allowed; @@ -963,6 +1054,7 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { if (!F.isIntrinsic()) { A.getOrCreateAAFor(IRPosition::function(F)); A.getOrCreateAAFor(IRPosition::function(F)); + A.getOrCreateAAFor(IRPosition::function(F)); CallingConv::ID CC = F.getCallingConv(); if (!AMDGPU::isEntryFunctionCC(CC)) { A.getOrCreateAAFor(IRPosition::function(F)); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 2569f40fec0e..12433dc83c48 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -748,35 +748,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields( } bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const { - for (const BasicBlock &BB : F) { - for (const Instruction &I : BB) { - const auto *CB = dyn_cast(&I); - if (!CB) - continue; - - if (CB->isInlineAsm()) { - const InlineAsm *IA = dyn_cast(CB->getCalledOperand()); - for (const auto &CI : IA->ParseConstraints()) { - for (StringRef Code : CI.Codes) { - Code.consume_front("{"); - if (Code.starts_with("a")) - return true; - } - } - continue; - } - - const Function *Callee = - dyn_cast(CB->getCalledOperand()->stripPointerCasts()); - if (!Callee) - return true; - - if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic) - return true; - } - } - - return false; + return !F.hasFnAttribute("amdgpu-no-agpr"); } bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const { diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll index 66034af5c351..cff9ce050667 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll @@ -233,9 +233,9 @@ attributes #1 = { nounwind } ; AKF_HSA: attributes #[[ATTR1]] = { nounwind } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll new file mode 100644 index 000000000000..33b1cc65dc56 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll @@ -0,0 +1,255 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 4 +; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor %s | FileCheck %s + +define amdgpu_kernel void @kernel_uses_asm_virtreg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "a"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_virtreg_def() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg_def( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[DEF:%.*]] = call i32 asm sideeffect " +; CHECK-NEXT: ret void +; + %def = call i32 asm sideeffect "; def $0", "=a"() + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg_def_tuple() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg_def_tuple( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[DEF:%.*]] = call i64 asm sideeffect " +; CHECK-NEXT: ret void +; + %def = call i64 asm sideeffect "; def $0", "={a[0:1]}"() + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_virtreg_second_arg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg_second_arg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "v,a"(i32 poison, i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_non_agpr_asm() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_non_agpr_asm( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "v"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a0}"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg_tuple() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg_tuple( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a[0:1]}"(i64 poison) + ret void +} + +define void @func_uses_asm_virtreg_agpr() { +; CHECK-LABEL: define void @func_uses_asm_virtreg_agpr( +; CHECK-SAME: ) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "a"(i32 poison) + ret void +} + +define void @func_uses_asm_physreg_agpr() { +; CHECK-LABEL: define void @func_uses_asm_physreg_agpr( +; CHECK-SAME: ) #[[ATTR2]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a0}"(i32 poison) + ret void +} + +define void @func_uses_asm_physreg_agpr_tuple() { +; CHECK-LABEL: define void @func_uses_asm_physreg_agpr_tuple( +; CHECK-SAME: ) #[[ATTR2]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a[0:1]}"(i64 poison) + ret void +} + +declare void @unknown() + +define amdgpu_kernel void @kernel_calls_extern() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_extern( +; CHECK-SAME: ) #[[ATTR4:[0-9]+]] { +; CHECK-NEXT: call void @unknown() +; CHECK-NEXT: ret void +; + call void @unknown() + ret void +} + +define amdgpu_kernel void @kernel_calls_extern_marked_callsite() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_extern_marked_callsite( +; CHECK-SAME: ) #[[ATTR4]] { +; CHECK-NEXT: call void @unknown() #[[ATTR9:[0-9]+]] +; CHECK-NEXT: ret void +; + call void @unknown() #0 + ret void +} + +define amdgpu_kernel void @kernel_calls_indirect(ptr %indirect) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_indirect( +; CHECK-SAME: ptr [[INDIRECT:%.*]]) #[[ATTR4]] { +; CHECK-NEXT: call void [[INDIRECT]]() +; CHECK-NEXT: ret void +; + call void %indirect() + ret void +} + +define amdgpu_kernel void @kernel_calls_indirect_marked_callsite(ptr %indirect) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_indirect_marked_callsite( +; CHECK-SAME: ptr [[INDIRECT:%.*]]) #[[ATTR4]] { +; CHECK-NEXT: call void [[INDIRECT]]() #[[ATTR9]] +; CHECK-NEXT: ret void +; + call void %indirect() #0 + ret void +} + +define amdgpu_kernel void @kernel_transitively_uses_agpr_asm() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_transitively_uses_agpr_asm( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void @func_uses_asm_physreg_agpr() +; CHECK-NEXT: ret void +; + call void @func_uses_asm_physreg_agpr() + ret void +} + +define void @empty() { +; CHECK-LABEL: define void @empty( +; CHECK-SAME: ) #[[ATTR5:[0-9]+]] { +; CHECK-NEXT: ret void +; + ret void +} + +define void @also_empty() { +; CHECK-LABEL: define void @also_empty( +; CHECK-SAME: ) #[[ATTR5]] { +; CHECK-NEXT: ret void +; + ret void +} + +define amdgpu_kernel void @kernel_calls_empty() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_empty( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: call void @empty() +; CHECK-NEXT: ret void +; + call void @empty() + ret void +} + +define amdgpu_kernel void @kernel_calls_non_agpr_and_agpr() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_non_agpr_and_agpr( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void @empty() +; CHECK-NEXT: call void @func_uses_asm_physreg_agpr() +; CHECK-NEXT: ret void +; + call void @empty() + call void @func_uses_asm_physreg_agpr() + ret void +} + +define amdgpu_kernel void @kernel_calls_generic_intrinsic(ptr %ptr0, ptr %ptr1, i64 %size) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_generic_intrinsic( +; CHECK-SAME: ptr [[PTR0:%.*]], ptr [[PTR1:%.*]], i64 [[SIZE:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[PTR0]], ptr [[PTR1]], i64 [[SIZE]], i1 false) +; CHECK-NEXT: ret void +; + call void @llvm.memcpy.p0.p0.i64(ptr %ptr0, ptr %ptr1, i64 %size, i1 false) + ret void +} + +declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) + +define amdgpu_kernel void @kernel_calls_mfma.f32.32x32x1f32(ptr addrspace(1) %out, float %a, float %b, <32 x float> %c) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_mfma.f32.32x32x1f32( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[A:%.*]], float [[B:%.*]], <32 x float> [[C:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RESULT:%.*]] = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float [[A]], float [[B]], <32 x float> [[C]], i32 0, i32 0, i32 0) +; CHECK-NEXT: store <32 x float> [[RESULT]], ptr addrspace(1) [[OUT]], align 128 +; CHECK-NEXT: ret void +; + %result = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %a, float %b, <32 x float> %c, i32 0, i32 0, i32 0) + store <32 x float> %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @kernel_calls_workitem_id_x(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_workitem_id_x( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: store i32 [[RESULT]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: ret void +; + %result = call i32 @llvm.amdgcn.workitem.id.x() + store i32 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @indirect_calls_none_agpr(i1 %cond) { +; CHECK-LABEL: define amdgpu_kernel void @indirect_calls_none_agpr( +; CHECK-SAME: i1 [[COND:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[FPTR:%.*]] = select i1 [[COND]], ptr @empty, ptr @also_empty +; CHECK-NEXT: call void [[FPTR]]() +; CHECK-NEXT: ret void +; + %fptr = select i1 %cond, ptr @empty, ptr @also_empty + call void %fptr() + ret void +} + + +attributes #0 = { "amdgpu-no-agpr" } +;. +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3:[0-9]+]] = { "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6:[0-9]+]] = { convergent nocallback nofree nosync nounwind willreturn memory(none) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR8:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" } +;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll index af0eb23d8e99..3d4ae84d9c69 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll @@ -1025,33 +1025,33 @@ attributes #6 = { "enqueued-block" } ; AKF_HSA: attributes #[[ATTR8]] = { "amdgpu-calls" } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR16]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR18]] = { nounwind "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR23:[0-9]+]] = { nounwind sanitize_address "amdgpu-no-implicitarg-ptr" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR24:[0-9]+]] = { "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR26]] = { "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR27]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR27]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR28]] = { nounwind } ; ATTRIBUTOR_HSA: attributes #[[ATTR29]] = { "enqueued-block" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll index 9a9c28ac632f..43cdf85ed381 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll @@ -643,19 +643,19 @@ attributes #1 = { nounwind } ; AKF_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-stack-objects" } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll index 6c5e58c74033..547ff69592ca 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll @@ -393,17 +393,18 @@ define amdgpu_kernel void @use_get_local_size_z(ptr addrspace(1) %ptr) #1 { attributes #0 = { nounwind readnone } attributes #1 = { nounwind } +;. ; AKF_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ; AKF_CHECK: attributes #[[ATTR1]] = { nounwind } ;. ; ATTRIBUTOR_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir b/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir index 895185cb41a3..577d38e65668 100644 --- a/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir +++ b/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir @@ -333,7 +333,7 @@ ret void } - attributes #0 = { "amdgpu-waves-per-eu"="4,4" } + attributes #0 = { "amdgpu-waves-per-eu"="4,4" "amdgpu-no-agpr" } ... --- diff --git a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll index 0c034192869b..386f9cd3f9ce 100644 --- a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll @@ -35,6 +35,6 @@ define amdgpu_kernel void @test_direct_indirect_call() { ret void } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR1]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll index 0069370cc972..05558c555c58 100644 --- a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll +++ b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll @@ -42,6 +42,6 @@ attributes #0 = { "amdgpu-no-dispatch-id" } ;. ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-no-dispatch-id" "amdgpu-stack-objects" } ;. -; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-dispatch-id" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll index a5792bf29ddc..4c21f8729745 100644 --- a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll +++ b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll @@ -258,25 +258,25 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo ;. ; V4: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V4: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V5: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V5: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V6: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V6: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V4: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll index e7488e059ee9..20edbd6c0d0f 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll @@ -157,27 +157,27 @@ define amdgpu_kernel void @test_preload_hint_kernel_1_call_func(ptr %0) #0 { define amdgpu_kernel void @test_preload_hint_kernel_1_call_intrinsic(i16 %0) #0 { ; NO-PRELOAD-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; NO-PRELOAD-SAME: (i16 [[TMP0:%.*]]) #[[ATTR2]] { +; NO-PRELOAD-SAME: (i16 [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; NO-PRELOAD-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; NO-PRELOAD-NEXT: ret void ; ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-1-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-1-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-1-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-1-NEXT: ret void ; ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-3-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-3-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-3-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-3-NEXT: ret void ; ; PRELOAD-16-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-16-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-16-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-16-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-16-NEXT: ret void ; ; PRELOAD-20-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-20-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-20-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-20-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-20-NEXT: ret void ; @@ -235,23 +235,23 @@ define amdgpu_kernel void @test_preload_hint_kernel_2_preexisting(i32 inreg %0, define amdgpu_kernel void @test_preload_hint_kernel_incompatible_attributes(ptr addrspace(4) byref(i32) %0, ptr nest %1) { ; NO-PRELOAD-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; NO-PRELOAD-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; NO-PRELOAD-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; NO-PRELOAD-NEXT: ret void ; ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-1-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-1-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-1-NEXT: ret void ; ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-3-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-3-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-3-NEXT: ret void ; ; PRELOAD-16-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-16-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-16-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-16-NEXT: ret void ; ; PRELOAD-20-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-20-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-20-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-20-NEXT: ret void ; ret void diff --git a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll index d92ba7774bd3..d070dc3b770f 100644 --- a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll @@ -203,13 +203,13 @@ attributes #5 = { "amdgpu-flat-work-group-size"="128,512" } attributes #6 = { "amdgpu-flat-work-group-size"="512,512" } attributes #7 = { "amdgpu-flat-work-group-size"="64,256" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll index 2df219bd0401..f62f1d57aec8 100644 --- a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll @@ -399,26 +399,26 @@ attributes #17 = { "amdgpu-waves-per-eu"="5,8" } attributes #18 = { "amdgpu-waves-per-eu"="9,10" } attributes #19 = { "amdgpu-waves-per-eu"="8,9" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll index eaef63bbfc3c..c1d647c5d3b9 100644 --- a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll +++ b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll @@ -19,5 +19,5 @@ define void @hoge() { ret void } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll index 297a056526ca..384a9c4043a1 100644 --- a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll @@ -191,11 +191,11 @@ define amdgpu_kernel void @kernel_lds_recursion() { !1 = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR3]] = { "amdgpu-lds-size"="4" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR5:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) } ; CHECK: attributes #[[ATTR6:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ;. diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll index f229f33664e1..539cfc71a80f 100644 --- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll @@ -73,7 +73,7 @@ define amdgpu_kernel void @test_simple_indirect_call() { ;. ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-stack-objects" } ;. -; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll index 8d5dc7943164..049db01badac 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll @@ -31,6 +31,6 @@ define amdgpu_kernel void @kernel1() #1 { attributes #0 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll index 7a6f82d589e6..c9387f196dff 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll @@ -98,7 +98,7 @@ define amdgpu_kernel void @kernel2() #0 { attributes #0 = { "uniform-work-group-size"="true" } ;. ; CHECK: attributes #[[ATTR0]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR2]] = { "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll index c04154c7c23f..7183da2c5efc 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll @@ -41,6 +41,6 @@ define amdgpu_kernel void @kernel3() #2 { attributes #2 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll index 2d5ff045d12e..6ed04cf63d20 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll @@ -41,7 +41,7 @@ define amdgpu_kernel void @kernel2() #2 { attributes #1 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll index e8bf6fc8321b..d5ba2fd617c6 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll @@ -52,8 +52,8 @@ attributes #0 = { nounwind } attributes #1 = { "uniform-work-group-size"="false" } attributes #2 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR3]] = { "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll index 473eea4eedce..7f0dfeaf75c8 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll @@ -101,7 +101,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %m) #1 { attributes #0 = { nounwind readnone } attributes #1 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll index 221f1a11676f..8616c73ad51c 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll @@ -61,6 +61,6 @@ define amdgpu_kernel void @kernel3() #0 { attributes #0 = { "uniform-work-group-size"="false" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll index 717d3d975aaf..040799435db4 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll @@ -540,6 +540,7 @@ define internal void @use512vgprs() { } define void @foo() #0 { + call void asm sideeffect "; use $0", "a"(i32 0) ret void } -- GitLab From b433076fcbacba8a3b91446390bbea5843322bcd Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Thu, 7 Mar 2024 07:49:40 +0100 Subject: [PATCH 015/933] [clang][CodeGen] Allow `memcpy` replace with trivial auto var init When emitting the storage (or memory copy operations) for constant initializers, the decision whether to split a constant structure or array store into a sequence of field stores or to use `memcpy` is based upon the optimization level and the size of the initializer. In afe8b93ffdfef5d8879e1894b9d7dda40dee2b8d, we extended this by allowing constants to be split when the array (or struct) type does not match the type of data the address to the object (constant) is expected to contain. This may happen when `emitStoresForConstant` is called by `EmitAutoVarInit`, as the element type of the address gets shrunk. When this occurs, let the initializer be split into a bunch of stores only under `-ftrivial-auto-var-init=pattern`. Fixes: https://github.com/llvm/llvm-project/issues/84178. --- clang/lib/CodeGen/CGDecl.cpp | 43 ++++++++++++++--------- clang/test/CodeGen/aapcs-align.cpp | 4 +-- clang/test/CodeGen/aapcs64-align.cpp | 8 ++--- clang/test/CodeGen/attr-counted-by.c | 26 ++++---------- clang/test/CodeGenCXX/auto-var-init.cpp | 27 +++++++------- clang/test/CodeGenOpenCL/amdgpu-printf.cl | 9 +---- clang/test/OpenMP/bug54082.c | 4 +-- 7 files changed, 56 insertions(+), 65 deletions(-) diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index dc42faf8dbb9..2ef5ed04af30 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1242,27 +1242,38 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D, return; } - // If the initializer is small, use a handful of stores. + // If the initializer is small or trivialAutoVarInit is set, use a handful of + // stores. + bool IsTrivialAutoVarInitPattern = + CGM.getContext().getLangOpts().getTrivialAutoVarInit() == + LangOptions::TrivialAutoVarInitKind::Pattern; if (shouldSplitConstantStore(CGM, ConstantSize)) { if (auto *STy = dyn_cast(Ty)) { - const llvm::StructLayout *Layout = - CGM.getDataLayout().getStructLayout(STy); - for (unsigned i = 0; i != constant->getNumOperands(); i++) { - CharUnits CurOff = CharUnits::fromQuantity(Layout->getElementOffset(i)); - Address EltPtr = Builder.CreateConstInBoundsByteGEP( - Loc.withElementType(CGM.Int8Ty), CurOff); - emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, - constant->getAggregateElement(i), IsAutoInit); + if (STy == Loc.getElementType() || + (STy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) { + const llvm::StructLayout *Layout = + CGM.getDataLayout().getStructLayout(STy); + for (unsigned i = 0; i != constant->getNumOperands(); i++) { + CharUnits CurOff = + CharUnits::fromQuantity(Layout->getElementOffset(i)); + Address EltPtr = Builder.CreateConstInBoundsByteGEP( + Loc.withElementType(CGM.Int8Ty), CurOff); + emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, + constant->getAggregateElement(i), IsAutoInit); + } + return; } - return; } else if (auto *ATy = dyn_cast(Ty)) { - for (unsigned i = 0; i != ATy->getNumElements(); i++) { - Address EltPtr = Builder.CreateConstGEP( - Loc.withElementType(ATy->getElementType()), i); - emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, - constant->getAggregateElement(i), IsAutoInit); + if (ATy == Loc.getElementType() || + (ATy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) { + for (unsigned i = 0; i != ATy->getNumElements(); i++) { + Address EltPtr = Builder.CreateConstGEP( + Loc.withElementType(ATy->getElementType()), i); + emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, + constant->getAggregateElement(i), IsAutoInit); + } + return; } - return; } } diff --git a/clang/test/CodeGen/aapcs-align.cpp b/clang/test/CodeGen/aapcs-align.cpp index 2886a32974b0..4f393d9e6b7f 100644 --- a/clang/test/CodeGen/aapcs-align.cpp +++ b/clang/test/CodeGen/aapcs-align.cpp @@ -134,8 +134,8 @@ void g6() { f6m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g6 -// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 0]) -// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 0]) +// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 undef]) +// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 undef]) // CHECK: declare void @f6(i32 noundef, [4 x i32]) // CHECK: declare void @f6m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [4 x i32]) } diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp index 759413cbc4b5..de231f2123b9 100644 --- a/clang/test/CodeGen/aapcs64-align.cpp +++ b/clang/test/CodeGen/aapcs64-align.cpp @@ -75,8 +75,8 @@ void g4() { f4m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g4() -// CHECK: call void @f4(i32 noundef 1, [2 x i64] %{{.*}}) -// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}}) +// CHECK: call void @f4(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0]) +// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0]) // CHECK: declare void @f4(i32 noundef, [2 x i64]) // CHECK: declare void @f4m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64]) @@ -95,8 +95,8 @@ void f5m(int, int, int, int, int, P16); f5m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g5() -// CHECK: call void @f5(i32 noundef 1, [2 x i64] %{{.*}}) -// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}}) +// CHECK: call void @f5(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0]) +// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0]) // CHECK: declare void @f5(i32 noundef, [2 x i64]) // CHECK: declare void @f5m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64]) diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c index e5685e39173b..1fb39f9a3466 100644 --- a/clang/test/CodeGen/attr-counted-by.c +++ b/clang/test/CodeGen/attr-counted-by.c @@ -1314,17 +1314,10 @@ int test14(int idx) { // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15( // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR4]] { // NO-SANITIZE-WITH-ATTR-NEXT: entry: -// NO-SANITIZE-WITH-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]] -// NO-SANITIZE-WITH-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4 -// NO-SANITIZE-WITH-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: [[BLAH:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 8 // NO-SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64 -// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[BLAH]], i64 0, i64 [[IDXPROM]] -// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] -// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]] -// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP1]] +// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]] +// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] +// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP0]] // // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15( // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] { @@ -1342,17 +1335,10 @@ int test14(int idx) { // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15( // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] { // NO-SANITIZE-WITHOUT-ATTR-NEXT: entry: -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[BLAH:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 8 // NO-SANITIZE-WITHOUT-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[BLAH]], i64 0, i64 [[IDXPROM]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP1]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP0]] // int test15(int idx) { struct { diff --git a/clang/test/CodeGenCXX/auto-var-init.cpp b/clang/test/CodeGenCXX/auto-var-init.cpp index 991eb73fe45c..7803ed5b633f 100644 --- a/clang/test/CodeGenCXX/auto-var-init.cpp +++ b/clang/test/CodeGenCXX/auto-var-init.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK,CHECK-O0 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0 -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,PATTERN,PATTERN-O1 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=PATTERN,PATTERN-O1 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,ZERO,ZERO-O0 -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,ZERO,ZERO-O1 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=ZERO,ZERO-O1 // RUN: %clang_cc1 -std=c++14 -triple i386-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0 #pragma clang diagnostic ignored "-Winaccessible-base" @@ -1303,9 +1303,10 @@ TEST_CUSTOM(semivolatile, semivolatile, { 0x44444444, 0x44444444 }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 1145324612, ptr %custom, align 4 -// CHECK-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4 -// CHECK-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4 +// PATTERN-O1: store i32 1145324612, ptr %custom, align 4 +// PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4 +// PATTERN-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4 +// ZERO-O1: store i64 4919131752989213764, ptr %custom, align 8 // CHECK-NOT: !annotation TEST_UNINIT(semivolatileinit, semivolatileinit); @@ -1418,7 +1419,8 @@ TEST_CUSTOM(matching, matching, { .f = 0xf00f }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store float 6.145500e+04, ptr {{.*}}, align 4 +// PATTERN-O1: store float 6.145500e+04, ptr {{.*}}, align 4 +// ZERO-O1: store i32 1198526208, ptr %custom, align 4 // CHECK-NOT: !annotation TEST_UNINIT(matchingreverse, matchingreverse); @@ -1445,7 +1447,8 @@ TEST_CUSTOM(matchingreverse, matchingreverse, { .i = 0xf00f }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 61455, ptr %custom, align 4 +// PATTERN-O1: store i32 61455, ptr %custom, align 4 +// ZERO-O1: store i32 61455, ptr %custom, align 4 // CHECK-NOT: !annotation TEST_UNINIT(unmatched, unmatched); @@ -1471,7 +1474,8 @@ TEST_CUSTOM(unmatched, unmatched, { .i = 0x3badbeef }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 1001242351, ptr {{.*}}, align 4 +// PATTERN-O1: store i32 1001242351, ptr {{.*}}, align 4 +// ZERO-O1: store i32 1001242351, ptr {{.*}}, align 4 // CHECK-NOT: !annotation TEST_UNINIT(unmatchedreverse, unmatchedreverse); @@ -1504,9 +1508,7 @@ TEST_CUSTOM(unmatchedreverse, unmatchedreverse, { .c = 42 }); // PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}} // PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 3 // PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}} -// ZERO-O1: store i8 42, ptr {{.*}}, align 4 -// ZERO-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 1 -// ZERO-O1-NEXT: call void @llvm.memset.{{.*}}({{.*}}, i8 0, i64 3, {{.*}}) +// ZERO-O1: store i32 42, ptr {{.*}}, align 4 TEST_UNINIT(unmatchedfp, unmatchedfp); // CHECK-LABEL: @test_unmatchedfp_uninit() @@ -1531,7 +1533,8 @@ TEST_CUSTOM(unmatchedfp, unmatchedfp, { .d = 3.1415926535897932384626433 }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store double 0x400921FB54442D18, ptr %custom, align 8 +// PATTERN-O1: store double 0x400921FB54442D18, ptr %custom, align 8 +// ZERO-O1: store i64 4614256656552045848, ptr %custom, align 8 // CHECK-NOT: !annotation TEST_UNINIT(emptyenum, emptyenum); diff --git a/clang/test/CodeGenOpenCL/amdgpu-printf.cl b/clang/test/CodeGenOpenCL/amdgpu-printf.cl index 6c84485b66b4..edf6dbf8657c 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-printf.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-printf.cl @@ -30,14 +30,7 @@ __kernel void test_printf_int(int i) { // CHECK-NEXT: [[S:%.*]] = alloca [4 x i8], align 1, addrspace(5) // CHECK-NEXT: store i32 [[I:%.*]], ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]] // CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[S]]) #[[ATTR5:[0-9]+]] -// CHECK-NEXT: [[LOC0:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 0 -// CHECK-NEXT: store i8 102, ptr addrspace(5) [[LOC0]], align 1 -// CHECK-NEXT: [[LOC1:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 1 -// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC1]], align 1 -// CHECK-NEXT: [[LOC2:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 2 -// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC2]], align 1 -// CHECK-NEXT: [[LOC3:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 3 -// CHECK-NEXT: store i8 0, ptr addrspace(5) [[LOC3]], align 1 +// CHECK-NEXT: call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 1 [[S]], ptr addrspace(4) align 1 @__const.test_printf_str_int.s, i64 4, i1 false) // CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(5) [[S]], i64 0, i64 0 // CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]] // CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr addrspace(4), ...) @printf(ptr addrspace(4) noundef @.str.2, ptr addrspace(5) noundef [[ARRAYDECAY]], i32 noundef [[TMP2]]) #[[ATTR4]] diff --git a/clang/test/OpenMP/bug54082.c b/clang/test/OpenMP/bug54082.c index b88b68fd4301..337c120983e0 100644 --- a/clang/test/OpenMP/bug54082.c +++ b/clang/test/OpenMP/bug54082.c @@ -69,9 +69,7 @@ void foo() { // CHECK-NEXT: [[X_TRAITS:%.*]] = alloca [1 x %struct.omp_alloctrait_t], align 16 // CHECK-NEXT: [[X_ALLOC:%.*]] = alloca i64, align 8 // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X_TRAITS]]) #[[ATTR5:[0-9]+]] -// CHECK-NEXT: store i32 2, ptr [[X_TRAITS]], align 16 -// CHECK-NEXT: [[LOC0:%.*]] = getelementptr inbounds i8, ptr [[X_TRAITS]], i64 8 -// CHECK-NEXT: store i64 64, ptr [[LOC0]], align 8 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) [[X_TRAITS]], ptr noundef nonnull align 16 dereferenceable(16) @__const.foo.x_traits, i64 16, i1 false) // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[X_ALLOC]]) #[[ATTR5]] // CHECK-NEXT: [[CALL:%.*]] = call i64 @omp_init_allocator(i64 noundef 0, i32 noundef 1, ptr noundef nonnull [[X_TRAITS]]) #[[ATTR5]] // CHECK-NEXT: store i64 [[CALL]], ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3:![0-9]+]] -- GitLab From 9fb85b09946122aa5793b647d7939ac17817c5f5 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 10:07:03 +0100 Subject: [PATCH 016/933] [mlir][bazel] Update BUILD after 29bf32efbb646b2ab3dec25f100419fc75635878. --- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 4802daa66286..07c5a00c07d7 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -2585,6 +2585,7 @@ cc_library( hdrs = glob([ "include/llvm/Passes/*.h", "include/llvm/Passes/*.def", + "include/llvm/Passes/*.inc", ]) + ["include/llvm-c/Transforms/PassBuilder.h"], copts = llvm_copts, deps = [ -- GitLab From 7b5a5be2a7216906c20f9bcac2209ea3502a7a73 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:17:55 +0000 Subject: [PATCH 017/933] [DAG] visitSUB/visitSUBO - move getAsNonOpaqueConstant into the if() where its used. NFC. Noticed while beginning some cleanup for moving to pattern matchers --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f199625bf67a..188472219882 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3725,13 +3725,10 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue NewSel = foldBinOpIntoSelect(N)) return NewSel; - ConstantSDNode *N1C = getAsNonOpaqueConstant(N1); - // fold (sub x, c) -> (add x, -c) - if (N1C) { + if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1)) return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); - } if (isNullOrNullSplat(N0)) { unsigned BitWidth = VT.getScalarSizeInBits(); @@ -4131,13 +4128,11 @@ SDValue DAGCombiner::visitSUBO(SDNode *N) { return CombineTo(N, DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, CarryVT)); - ConstantSDNode *N1C = getAsNonOpaqueConstant(N1); - // fold (subox, c) -> (addo x, -c) - if (IsSigned && N1C && !N1C->isMinSignedValue()) { - return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, - DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); - } + if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1)) + if (IsSigned && !N1C->isMinSignedValue()) + return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, + DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); // fold (subo x, 0) -> x + no borrow if (isNullOrNullSplat(N1)) -- GitLab From ee5e027cc64957c0e18b8c38ce10d4b84314511c Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:29:39 +0000 Subject: [PATCH 018/933] [X86] getShuffleCost - recognise concat_vector(X,Y) shuffle as InsertSubvector instead of PermuteTwoSrc We don't have a concat_vector shuffle kind and improveShuffleKindFromMask won't alter the base type to match it as InsertSubvector. But since this is how X86 will lower concat_vector anyhow, just recognise it explicitly. Another step for #67803 --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 8 ++++ .../Transforms/PhaseOrdering/X86/pr67803.ll | 43 +++---------------- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index d336ab9d309c..e9bdc6ab5bea 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1480,6 +1480,14 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Kind = improveShuffleKindFromMask(Kind, Mask, BaseTp, Index, SubTp); + // Recognize a basic concat_vector shuffle. + if (Kind == TTI::SK_PermuteTwoSrc && + Mask.size() == (2 * BaseTp->getElementCount().getKnownMinValue()) && + ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) + return getShuffleCost(TTI::SK_InsertSubvector, + VectorType::getDoubleElementsVectorType(BaseTp), Mask, + CostKind, Mask.size() / 2, BaseTp); + // Treat Transpose as 2-op shuffles - there's no difference in lowering. if (Kind == TTI::SK_Transpose) Kind = TTI::SK_PermuteTwoSrc; diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll index e61b254b7a5f..495ec0a63399 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { ; CHECK-LABEL: @PR67803( @@ -11,16 +11,14 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> ; CHECK-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[SEXT_I22]] to <2 x i64> ; CHECK-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> ; CHECK-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[SEXT_I]] to <2 x i64> -; CHECK-NEXT: [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> ; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> ; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> poison, <16 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP9:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP11:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP6]], <16 x i8> [[TMP8]], <16 x i8> [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = bitcast <16 x i8> [[TMP11]] to <2 x i64> @@ -28,42 +26,13 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP15:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> poison, <16 x i32> -; CHECK-NEXT: [[TMP17:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> ; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP19:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP14]], <16 x i8> [[TMP16]], <16 x i8> [[TMP18]]) ; CHECK-NEXT: [[TMP20:%.*]] = bitcast <16 x i8> [[TMP19]] to <2 x i64> ; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] ; -; AVX512-LABEL: @PR67803( -; AVX512-NEXT: entry: -; AVX512-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] -; AVX512-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> -; AVX512-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> -; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> -; AVX512-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP6:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP10:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]]) -; AVX512-NEXT: [[TMP11:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64> -; AVX512-NEXT: [[TMP12:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> -; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP14:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> -; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP18:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP13]], <16 x i8> [[TMP15]], <16 x i8> [[TMP17]]) -; AVX512-NEXT: [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <2 x i64> -; AVX512-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP19]], <4 x i32> -; AVX512-NEXT: ret <4 x i64> [[SHUFFLE_I23]] -; entry: %0 = bitcast <4 x i64> %x to <8 x i32> %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> -- GitLab From a6a9215b93bcbf901cd11d2dd02cce1a245d3ffe Mon Sep 17 00:00:00 2001 From: Johannes Reifferscheid Date: Thu, 21 Mar 2024 10:33:49 +0100 Subject: [PATCH 019/933] Lower shuffle to single-result form if possible. (#84321) We currently always lower shuffle to the struct-returning variant. I saw some cases where this survived all the way through ptx, resulting in increased register usage. The easiest fix is to simply lower to the single-result version when the predicate is unused. --- .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp | 23 ++++++++---- .../Conversion/GPUToNVVM/gpu-to-nvvm.mlir | 36 +++++++++++++++++-- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp index d6a5d8cd74d5..b95fba20a00c 100644 --- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp +++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp @@ -155,8 +155,6 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern { auto valueTy = adaptor.getValue().getType(); auto int32Type = IntegerType::get(rewriter.getContext(), 32); auto predTy = IntegerType::get(rewriter.getContext(), 1); - auto resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), - {valueTy, predTy}); Value one = rewriter.create(loc, int32Type, 1); Value minusOne = rewriter.create(loc, int32Type, -1); @@ -176,14 +174,25 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern { rewriter.create(loc, int32Type, adaptor.getWidth(), one); } - auto returnValueAndIsValidAttr = rewriter.getUnitAttr(); + bool predIsUsed = !op->getResult(1).use_empty(); + UnitAttr returnValueAndIsValidAttr = nullptr; + Type resultTy = valueTy; + if (predIsUsed) { + returnValueAndIsValidAttr = rewriter.getUnitAttr(); + resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), + {valueTy, predTy}); + } Value shfl = rewriter.create( loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(), maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr); - Value shflValue = rewriter.create(loc, shfl, 0); - Value isActiveSrcLane = rewriter.create(loc, shfl, 1); - - rewriter.replaceOp(op, {shflValue, isActiveSrcLane}); + if (predIsUsed) { + Value shflValue = rewriter.create(loc, shfl, 0); + Value isActiveSrcLane = + rewriter.create(loc, shfl, 1); + rewriter.replaceOp(op, {shflValue, isActiveSrcLane}); + } else { + rewriter.replaceOp(op, {shfl, nullptr}); + } return success(); } }; diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir index dd3b6c2080aa..8877ee083286 100644 --- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir +++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir @@ -112,7 +112,7 @@ gpu.module @test_module_3 { gpu.module @test_module_4 { // CHECK-LABEL: func @gpu_shuffle() - func.func @gpu_shuffle() -> (f32, f32, f32, f32) { + func.func @gpu_shuffle() -> (f32, f32, f32, f32, i1, i1, i1, i1) { // CHECK: %[[#VALUE:]] = llvm.mlir.constant(1.000000e+00 : f32) : f32 %arg0 = arith.constant 1.0 : f32 // CHECK: %[[#OFFSET:]] = llvm.mlir.constant(4 : i32) : i32 @@ -143,11 +143,41 @@ gpu.module @test_module_4 { // CHECK: nvvm.shfl.sync idx {{.*}} {return_value_and_is_valid} : f32 -> !llvm.struct<(f32, i1)> %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32 - func.return %shfl, %shflu, %shfld, %shfli : f32, f32,f32, f32 + func.return %shfl, %shflu, %shfld, %shfli, %pred, %predu, %predd, %predi + : f32, f32,f32, f32, i1, i1, i1, i1 } -} + // CHECK-LABEL: func @gpu_shuffle_unused_pred() + func.func @gpu_shuffle_unused_pred() -> (f32, f32, f32, f32) { + // CHECK: %[[#VALUE:]] = llvm.mlir.constant(1.000000e+00 : f32) : f32 + %arg0 = arith.constant 1.0 : f32 + // CHECK: %[[#OFFSET:]] = llvm.mlir.constant(4 : i32) : i32 + %arg1 = arith.constant 4 : i32 + // CHECK: %[[#WIDTH:]] = llvm.mlir.constant(23 : i32) : i32 + %arg2 = arith.constant 23 : i32 + // CHECK: %[[#ONE:]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[#MINUS_ONE:]] = llvm.mlir.constant(-1 : i32) : i32 + // CHECK: %[[#THIRTY_TWO:]] = llvm.mlir.constant(32 : i32) : i32 + // CHECK: %[[#NUM_LANES:]] = llvm.sub %[[#THIRTY_TWO]], %[[#WIDTH]] : i32 + // CHECK: %[[#MASK:]] = llvm.lshr %[[#MINUS_ONE]], %[[#NUM_LANES]] : i32 + // CHECK: %[[#CLAMP:]] = llvm.sub %[[#WIDTH]], %[[#ONE]] : i32 + // CHECK: %[[#SHFL:]] = nvvm.shfl.sync bfly %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#CLAMP]] : f32 -> f32 + %shfl, %pred = gpu.shuffle xor %arg0, %arg1, %arg2 : f32 + // CHECK: %[[#ONE:]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[#MINUS_ONE:]] = llvm.mlir.constant(-1 : i32) : i32 + // CHECK: %[[#THIRTY_TWO:]] = llvm.mlir.constant(32 : i32) : i32 + // CHECK: %[[#NUM_LANES:]] = llvm.sub %[[#THIRTY_TWO]], %[[#WIDTH]] : i32 + // CHECK: %[[#MASK:]] = llvm.lshr %[[#MINUS_ONE]], %[[#NUM_LANES]] : i32 + // CHECK: %[[#SHFL:]] = nvvm.shfl.sync up %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#NUM_LANES]] : f32 -> f32 + %shflu, %predu = gpu.shuffle up %arg0, %arg1, %arg2 : f32 + // CHECK: nvvm.shfl.sync down {{.*}} : f32 -> f32 + %shfld, %predd = gpu.shuffle down %arg0, %arg1, %arg2 : f32 + // CHECK: nvvm.shfl.sync idx {{.*}} : f32 -> f32 + %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32 + func.return %shfl, %shflu, %shfld, %shfli : f32, f32,f32, f32 + } +} gpu.module @test_module_5 { // CHECK-LABEL: func @gpu_sync() -- GitLab From 11aa95f83b7bf980ea13f1bb75e09af89a733acb Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:46:31 +0000 Subject: [PATCH 020/933] [DAG] visitSUB - pull out repeated getScalarSizeInBits() calls. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 188472219882..d370c57ce8e3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3695,6 +3695,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); + unsigned BitWidth = VT.getScalarSizeInBits(); SDLoc DL(N); auto PeekThroughFreeze = [](SDValue N) { @@ -3731,7 +3732,6 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); if (isNullOrNullSplat(N0)) { - unsigned BitWidth = VT.getScalarSizeInBits(); // Right-shifting everything out but the sign bit followed by negation is // the same as flipping arithmetic/logical shift type without the negation: // -(X >>u 31) -> (X >>s 31) @@ -3931,7 +3931,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { SDValue S0 = N1.getOperand(0); if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1))) - if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) + if (C->getAPIntValue() == (BitWidth - 1)) return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); } } @@ -3974,8 +3974,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { SDValue ShAmt = N1.getOperand(1); ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt); - if (ShAmtC && - ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) { + if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) { SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); } @@ -3986,7 +3985,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { // N0 - (X << BW-1) --> N0 + (X << BW-1) if (N1.getOpcode() == ISD::SHL) { ConstantSDNode *ShlC = isConstOrConstSplat(N1.getOperand(1)); - if (ShlC && ShlC->getAPIntValue() == VT.getScalarSizeInBits() - 1) + if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1)) return DAG.getNode(ISD::ADD, DL, VT, N1, N0); } -- GitLab From 23de3862dce582ce91c1aa914467d982cb1a73b4 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:54:12 +0000 Subject: [PATCH 021/933] [DAG] visitSUB - use sd_match to match SUB(MAX,MIN) -> ABD pattern. NFC. Seriously simplifies the commutation matching logic. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index d370c57ce8e3..a3f5d433d920 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4018,23 +4018,17 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } } - // max(a,b) - min(a,b) --> abd(a,b) - auto MatchSubMaxMin = [&](unsigned Max, unsigned Min, unsigned Abd) { - if (N0.getOpcode() != Max || N1.getOpcode() != Min) - return SDValue(); - if ((N0.getOperand(0) != N1.getOperand(0) || - N0.getOperand(1) != N1.getOperand(1)) && - (N0.getOperand(0) != N1.getOperand(1) || - N0.getOperand(1) != N1.getOperand(0))) - return SDValue(); - if (!hasOperation(Abd, VT)) - return SDValue(); - return DAG.getNode(Abd, DL, VT, N0.getOperand(0), N0.getOperand(1)); - }; - if (SDValue R = MatchSubMaxMin(ISD::SMAX, ISD::SMIN, ISD::ABDS)) - return R; - if (SDValue R = MatchSubMaxMin(ISD::UMAX, ISD::UMIN, ISD::ABDU)) - return R; + // smax(a,b) - smin(a,b) --> abds(a,b) + if (hasOperation(ISD::ABDS, VT) && + sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && + sd_match(N1, m_SMin(m_Specific(A), m_Specific(B)))) + return DAG.getNode(ISD::ABDS, DL, VT, A, B); + + // umax(a,b) - umin(a,b) --> abdu(a,b) + if (hasOperation(ISD::ABDU, VT) && + sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && + sd_match(N1, m_UMin(m_Specific(A), m_Specific(B)))) + return DAG.getNode(ISD::ABDU, DL, VT, A, B); return SDValue(); } -- GitLab From ccb3a8feaa5b132dc829e55e069dde62008df4a8 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Thu, 21 Mar 2024 11:28:35 +0100 Subject: [PATCH 022/933] [AMDGPU][LowerModuleLDS] Refactor partially lowered module detection (#85793) Refactor the logic that checks if a module contains mixed absolute/non-lowered LDS GVs. The check now happens latter when the "worklists" are formed. This is because in some cases (OpenMP) we can have non-lowered GVs in a lowered module, and this is normal because those GVs are just unused and removed from the list at some point before the end of `getUsesOfLDSByFunction`. Doing the check later ensures that if a mixed module is spotted, then it's a _real_ mixed module that needs rejection, not a module containing an intentionally ignored GV. --- .../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 40 ++++++++++++------- .../lds-mixed-absolute-addresses-unused.ll | 26 ++++++++++++ .../lds-reject-mixed-absolute-addresses.ll | 2 +- 3 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp index b85cb26fdc95..595f09664c55 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp @@ -340,26 +340,11 @@ public: // Get uses from the current function, excluding uses by called functions // Two output variables to avoid walking the globals list twice - std::optional HasAbsoluteGVs; for (auto &GV : M.globals()) { if (!AMDGPU::isLDSVariableToLower(GV)) { continue; } - // Check if the module is consistent: either all GVs are absolute (happens - // when we run the pass more than once), or none are. - const bool IsAbsolute = GV.isAbsoluteSymbolRef(); - if (HasAbsoluteGVs.has_value()) { - if (*HasAbsoluteGVs != IsAbsolute) { - report_fatal_error( - "Module cannot mix absolute and non-absolute LDS GVs"); - } - } else - HasAbsoluteGVs = IsAbsolute; - - if (IsAbsolute) - continue; - for (User *V : GV.users()) { if (auto *I = dyn_cast(V)) { Function *F = I->getFunction(); @@ -469,6 +454,31 @@ public: } } + // Verify that we fall into one of 2 cases: + // - All variables are absolute: this is a re-run of the pass + // so we don't have anything to do. + // - No variables are absolute. + std::optional HasAbsoluteGVs; + for (auto &Map : {direct_map_kernel, indirect_map_kernel}) { + for (auto &[Fn, GVs] : Map) { + for (auto *GV : GVs) { + bool IsAbsolute = GV->isAbsoluteSymbolRef(); + if (HasAbsoluteGVs.has_value()) { + if (*HasAbsoluteGVs != IsAbsolute) { + report_fatal_error( + "Module cannot mix absolute and non-absolute LDS GVs"); + } + } else + HasAbsoluteGVs = IsAbsolute; + } + } + } + + // If we only had absolute GVs, we have nothing to do, return an empty + // result. + if (HasAbsoluteGVs && *HasAbsoluteGVs) + return {FunctionVariableMap(), FunctionVariableMap()}; + return {std::move(direct_map_kernel), std::move(indirect_map_kernel)}; } diff --git a/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll b/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll new file mode 100644 index 000000000000..d101d8da5e0f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s 2>&1 | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s 2>&1 | FileCheck %s + +; This looks like a partially lowered module, but the non-lowered GV isn't used by any kernels. +; In such cases, LowerModuleLDS is free to leave it in and ignore it, and we want to make sure +; LowerModuleLDS doesn't crash if it re-runs on such modules. +@notLowered = addrspace(3) global i32 poison +@lowered = addrspace(3) global i32 poison, !absolute_symbol !0 + +@llvm.compiler.used = appending addrspace(1) global [1 x ptr] [ptr addrspacecast (ptr addrspace(3) @notLowered to ptr)], section "llvm.metadata" + +define amdgpu_kernel void @kern(i32 %val0) { +; CHECK-LABEL: define amdgpu_kernel void @kern( +; CHECK-SAME: i32 [[VAL0:%.*]]) { +; CHECK-NEXT: [[VAL1:%.*]] = add i32 [[VAL0]], 4 +; CHECK-NEXT: store i32 [[VAL1]], ptr addrspace(3) @lowered, align 4 +; CHECK-NEXT: ret void +; + %val1 = add i32 %val0, 4 + store i32 %val1, ptr addrspace(3) @lowered + ret void +} + + +!0 = !{i32 0, i32 1} diff --git a/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll b/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll index b512a43aa102..b1f4f2ef1ef5 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll @@ -8,7 +8,7 @@ define amdgpu_kernel void @kern() { %val0 = load i32, ptr addrspace(3) @var1 %val1 = add i32 %val0, 4 - store i32 %val1, ptr addrspace(3) @var1 + store i32 %val1, ptr addrspace(3) @var2 ret void } -- GitLab From 8ecc377c88ba32978ace6a67de895403eeba3a22 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Thu, 21 Mar 2024 11:40:46 +0100 Subject: [PATCH 023/933] [llvm-lib] Use ARM64EC machine type for import libraries when -machine:arm64x is used. (#85972) This is compatible with MSVC, `-machine:arm64x` is essentially an alias to `-machine:arm64ec`. To make a type library that exposes both native and EC symbols, an additional `-defArm64Native` argument is needed in both cases. --- llvm/lib/Object/COFFImportFile.cpp | 7 +++++-- llvm/test/tools/llvm-lib/arm64ec-implib.test | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp index 46c8e702581e..8224a1492502 100644 --- a/llvm/lib/Object/COFFImportFile.cpp +++ b/llvm/lib/Object/COFFImportFile.cpp @@ -626,8 +626,11 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path, MachineTypes Machine, bool MinGW, ArrayRef NativeExports) { - MachineTypes NativeMachine = - isArm64EC(Machine) ? IMAGE_FILE_MACHINE_ARM64 : Machine; + MachineTypes NativeMachine = Machine; + if (isArm64EC(Machine)) { + NativeMachine = IMAGE_FILE_MACHINE_ARM64; + Machine = IMAGE_FILE_MACHINE_ARM64EC; + } std::vector Members; ObjectFactory OF(llvm::sys::path::filename(ImportName), NativeMachine); diff --git a/llvm/test/tools/llvm-lib/arm64ec-implib.test b/llvm/test/tools/llvm-lib/arm64ec-implib.test index 00eddd2a4752..9ce53fe0fea0 100644 --- a/llvm/test/tools/llvm-lib/arm64ec-implib.test +++ b/llvm/test/tools/llvm-lib/arm64ec-implib.test @@ -34,17 +34,17 @@ ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-readobj test.lib | FileCheck -check-prefix=READOBJ %s -READOBJ: File: test.lib(test.dll) +READOBJ: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit READOBJ-EMPTY: -READOBJ-NEXT: File: test.lib(test.dll) +READOBJ-NEXT: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit READOBJ-EMPTY: -READOBJ-NEXT: File: test.lib(test.dll) +READOBJ-NEXT: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit @@ -96,6 +96,11 @@ READOBJ-NEXT: Name type: name READOBJ-NEXT: Export name: dataexp READOBJ-NEXT: Symbol: __imp_dataexp +Using -machine:arm64x gives the same output. +RUN: llvm-lib -machine:arm64x -def:test.def -out:testx.lib +RUN: llvm-nm --print-armap testx.lib | FileCheck -check-prefix=ARMAP %s +RUN: llvm-readobj testx.lib | FileCheck -check-prefix=READOBJ %s + Creating a new lib containing the existing lib: RUN: llvm-lib -machine:arm64ec test.lib -out:test2.lib RUN: llvm-nm --print-armap test2.lib | FileCheck -check-prefix=ARMAP %s @@ -246,7 +251,9 @@ READOBJX-NEXT: Symbol: __imp_dataexp RUN: llvm-lib -machine:arm64ec -def:test.def -defArm64Native:test2.def -out:test2.lib +RUN: llvm-lib -machine:arm64ec -def:test.def -defArm64Native:test2.def -out:test2x.lib RUN: llvm-nm --print-armap test2.lib | FileCheck -check-prefix=ARMAPX2 %s +RUN: llvm-nm --print-armap test2x.lib | FileCheck -check-prefix=ARMAPX2 %s ARMAPX2: Archive map ARMAPX2-NEXT: __IMPORT_DESCRIPTOR_test2 in test2.dll -- GitLab From 0124e0821dd87bb021f31d1d9aecc2e0f3a52514 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Thu, 21 Mar 2024 11:43:48 +0100 Subject: [PATCH 024/933] [Object][COFF][NFC] Introduce Arm64ECThunkType enum. (#85936) And use it in EC lowering code. It will be useful for LLD too. --- llvm/include/llvm/BinaryFormat/COFF.h | 6 ++ .../AArch64/AArch64Arm64ECCallLowering.cpp | 57 ++++++++++--------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h index 72461d0d9c31..4c31cd847bdf 100644 --- a/llvm/include/llvm/BinaryFormat/COFF.h +++ b/llvm/include/llvm/BinaryFormat/COFF.h @@ -806,6 +806,12 @@ enum Feat00Flags : uint32_t { Kernel = 0x40000000, }; +enum class Arm64ECThunkType : uint8_t { + GuestExit = 0, + Entry = 1, + Exit = 4, +}; + inline bool isReservedSectionNumber(int32_t SectionNumber) { return SectionNumber <= 0; } diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp index f147ded2ab70..9b5cbe3de137 100644 --- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp @@ -30,6 +30,7 @@ #include "llvm/TargetParser/Triple.h" using namespace llvm; +using namespace llvm::COFF; using namespace llvm::object; using OperandBundleDef = OperandBundleDefT; @@ -45,8 +46,6 @@ static cl::opt GenerateThunks("arm64ec-generate-thunks", cl::Hidden, namespace { -enum class ThunkType { GuestExit, Entry, Exit }; - class AArch64Arm64ECCallLowering : public ModulePass { public: static char ID; @@ -73,15 +72,15 @@ private: Type *I64Ty; Type *VoidTy; - void getThunkType(FunctionType *FT, AttributeList AttrList, ThunkType TT, - raw_ostream &Out, FunctionType *&Arm64Ty, - FunctionType *&X64Ty); + void getThunkType(FunctionType *FT, AttributeList AttrList, + Arm64ECThunkType TT, raw_ostream &Out, + FunctionType *&Arm64Ty, FunctionType *&X64Ty); void getThunkRetType(FunctionType *FT, AttributeList AttrList, raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool &HasSretPtr); - void getThunkArgTypes(FunctionType *FT, AttributeList AttrList, ThunkType TT, - raw_ostream &Out, + void getThunkArgTypes(FunctionType *FT, AttributeList AttrList, + Arm64ECThunkType TT, raw_ostream &Out, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool HasSretPtr); void canonicalizeThunkType(Type *T, Align Alignment, bool Ret, @@ -91,13 +90,11 @@ private: } // end anonymous namespace -void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, - AttributeList AttrList, - ThunkType TT, raw_ostream &Out, - FunctionType *&Arm64Ty, - FunctionType *&X64Ty) { - Out << (TT == ThunkType::Entry ? "$ientry_thunk$cdecl$" - : "$iexit_thunk$cdecl$"); +void AArch64Arm64ECCallLowering::getThunkType( + FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT, + raw_ostream &Out, FunctionType *&Arm64Ty, FunctionType *&X64Ty) { + Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$" + : "$iexit_thunk$cdecl$"); Type *Arm64RetTy; Type *X64RetTy; @@ -108,7 +105,7 @@ void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, // The first argument to a thunk is the called function, stored in x9. // For exit thunks, we pass the called function down to the emulator; // for entry/guest exit thunks, we just call the Arm64 function directly. - if (TT == ThunkType::Exit) + if (TT == Arm64ECThunkType::Exit) Arm64ArgTypes.push_back(PtrTy); X64ArgTypes.push_back(PtrTy); @@ -125,8 +122,8 @@ void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, } void AArch64Arm64ECCallLowering::getThunkArgTypes( - FunctionType *FT, AttributeList AttrList, ThunkType TT, raw_ostream &Out, - SmallVectorImpl &Arm64ArgTypes, + FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT, + raw_ostream &Out, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool HasSretPtr) { Out << "$"; @@ -163,7 +160,7 @@ void AArch64Arm64ECCallLowering::getThunkArgTypes( X64ArgTypes.push_back(PtrTy); // x5 Arm64ArgTypes.push_back(I64Ty); - if (TT != ThunkType::Entry) { + if (TT != Arm64ECThunkType::Entry) { // FIXME: x5 isn't actually used by the x64 side; revisit once we // have proper isel for varargs X64ArgTypes.push_back(I64Ty); @@ -348,7 +345,8 @@ Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT, SmallString<256> ExitThunkName; llvm::raw_svector_ostream ExitThunkStream(ExitThunkName); FunctionType *Arm64Ty, *X64Ty; - getThunkType(FT, Attrs, ThunkType::Exit, ExitThunkStream, Arm64Ty, X64Ty); + getThunkType(FT, Attrs, Arm64ECThunkType::Exit, ExitThunkStream, Arm64Ty, + X64Ty); if (Function *F = M->getFunction(ExitThunkName)) return F; @@ -451,8 +449,8 @@ Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) { SmallString<256> EntryThunkName; llvm::raw_svector_ostream EntryThunkStream(EntryThunkName); FunctionType *Arm64Ty, *X64Ty; - getThunkType(F->getFunctionType(), F->getAttributes(), ThunkType::Entry, - EntryThunkStream, Arm64Ty, X64Ty); + getThunkType(F->getFunctionType(), F->getAttributes(), + Arm64ECThunkType::Entry, EntryThunkStream, Arm64Ty, X64Ty); if (Function *F = M->getFunction(EntryThunkName)) return F; @@ -543,8 +541,8 @@ Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) { Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) { llvm::raw_null_ostream NullThunkName; FunctionType *Arm64Ty, *X64Ty; - getThunkType(F->getFunctionType(), F->getAttributes(), ThunkType::GuestExit, - NullThunkName, Arm64Ty, X64Ty); + getThunkType(F->getFunctionType(), F->getAttributes(), + Arm64ECThunkType::GuestExit, NullThunkName, Arm64Ty, X64Ty); auto MangledName = getArm64ECMangledFunctionName(F->getName().str()); assert(MangledName && "Can't guest exit to function that's already native"); std::string ThunkName = *MangledName; @@ -679,7 +677,7 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { struct ThunkInfo { Constant *Src; Constant *Dst; - unsigned Kind; + Arm64ECThunkType Kind; }; SmallVector ThunkMapping; for (Function &F : Mod) { @@ -688,14 +686,17 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) { if (!F.hasComdat()) F.setComdat(Mod.getOrInsertComdat(F.getName())); - ThunkMapping.push_back({&F, buildEntryThunk(&F), 1}); + ThunkMapping.push_back( + {&F, buildEntryThunk(&F), Arm64ECThunkType::Entry}); } } for (Function *F : DirectCalledFns) { ThunkMapping.push_back( - {F, buildExitThunk(F->getFunctionType(), F->getAttributes()), 4}); + {F, buildExitThunk(F->getFunctionType(), F->getAttributes()), + Arm64ECThunkType::Exit}); if (!F->hasDLLImportStorageClass()) - ThunkMapping.push_back({buildGuestExitThunk(F), F, 0}); + ThunkMapping.push_back( + {buildGuestExitThunk(F), F, Arm64ECThunkType::GuestExit}); } if (!ThunkMapping.empty()) { @@ -704,7 +705,7 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { ThunkMappingArrayElems.push_back(ConstantStruct::getAnon( {ConstantExpr::getBitCast(Thunk.Src, PtrTy), ConstantExpr::getBitCast(Thunk.Dst, PtrTy), - ConstantInt::get(M->getContext(), APInt(32, Thunk.Kind))})); + ConstantInt::get(M->getContext(), APInt(32, uint8_t(Thunk.Kind)))})); } Constant *ThunkMappingArray = ConstantArray::get( llvm::ArrayType::get(ThunkMappingArrayElems[0]->getType(), -- GitLab From 95a834a16c3de0de615d0cfa20a6c8bd973b6a1d Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Thu, 21 Mar 2024 11:44:47 +0100 Subject: [PATCH 025/933] (Reland) [AMDGPU] Run LowerLDS at the end of the fullLTO pipeline (#85626) Reland of #75333 --- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 ++++ .../CodeGen/AMDGPU/lto-lower-module-lds.ll | 47 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 2b457fe519d9..c96625092a76 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -793,6 +793,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( PM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); }); + + PB.registerFullLinkTimeOptimizationLastEPCallback( + [this](ModulePassManager &PM, OptimizationLevel Level) { + // We want to support the -lto-partitions=N option as "best effort". + // For that, we need to lower LDS earlier in the pipeline before the + // module is partitioned for codegen. + if (EnableLowerModuleLDS) + PM.addPass(AMDGPULowerModuleLDSPass(*this)); + }); } int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) { diff --git a/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll new file mode 100644 index 000000000000..f1d946376afe --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll @@ -0,0 +1,47 @@ + +; Default O0 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O0 -cg-opt-level 0 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O0 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O0 -cg-opt-level 0 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O1 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O1 -cg-opt-level 1 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O1 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O1 -cg-opt-level 1 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O2 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O2 -cg-opt-level 2 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O2 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O2 -cg-opt-level 2 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O3 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O3 -cg-opt-level 3 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O3 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O3 -cg-opt-level 3 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; First print will be from the New PM during the full LTO pipeline. +; Second print will be from the legacy PM during the CG pipeline. + +; CHECK: Running pass: AMDGPULowerModuleLDSPass on [module] +; CHECK: ModulePass Manager +; CHECK: Lower uses of LDS variables from non-kernel functions + +@lds = internal unnamed_addr addrspace(3) global i32 poison, align 4 + +define amdgpu_kernel void @test() { +entry: + store i32 1, ptr addrspace(3) @lds + ret void +} -- GitLab From 2096f37d7a580a4b4ddce2a44abb80ff90f273ee Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Thu, 21 Mar 2024 11:57:07 +0100 Subject: [PATCH 026/933] [libc++][NFC] Use __constexpr_memmove instead of copy_n in <__string/char_traits.h> (#85920) `copy_n` has been used to allow constant evaluation of `char_traits`. We now have `__constexpr_memmove`, which `copy_n` just forwards to. We can call `__constexpr_memmove` directly, avoiding a bunch of instantiations. This reduces the time it takes to include `` from 321ms to 285ms. --- libcxx/include/__string/char_traits.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libcxx/include/__string/char_traits.h b/libcxx/include/__string/char_traits.h index 5880d3a22db2..47ed1057caaa 100644 --- a/libcxx/include/__string/char_traits.h +++ b/libcxx/include/__string/char_traits.h @@ -9,7 +9,6 @@ #ifndef _LIBCPP___STRING_CHAR_TRAITS_H #define _LIBCPP___STRING_CHAR_TRAITS_H -#include <__algorithm/copy_n.h> #include <__algorithm/fill_n.h> #include <__algorithm/find_end.h> #include <__algorithm/find_first_of.h> @@ -144,7 +143,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -221,7 +220,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -287,7 +286,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -366,7 +365,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -454,7 +453,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static char_type* copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } -- GitLab From cb071942f881e743b8131688a873dab760c7b88d Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Thu, 21 Mar 2024 12:01:26 +0100 Subject: [PATCH 027/933] [OpenMP] Fix SystemZ build failure Commit a7d5f73a03c81cab8df64dbd099e8acb40f5dfe1 introduced an error in a target_compile_definitions on the SystemZ, causing the build to break. Fixed by adding the missing "PRIVATE". --- openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index 5ccb20e305e8..58a79898ff80 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -100,7 +100,7 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$") "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO") set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$") - target_compile_definitions(omptarget.rtl.${machine} TARGET_ELF_ID=EM_S390) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_S390) target_compile_definitions(omptarget.rtl.${machine} PRIVATE LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="s390x-ibm-linux-gnu") list(APPEND LIBOMPTARGET_SYSTEM_TARGETS -- GitLab From e8cf1754988cf281e76ae463b3e1f3c0cda3f230 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Thu, 21 Mar 2024 12:05:11 +0100 Subject: [PATCH 028/933] [runtimes] Fix OpenMP dependencies (#85977) When building the OpenMP runtime with libomptarget support, the runtimes configure step needs to have a dependency on various tools, in particular opt, so that cmake configure checks yield the correct results. This did not work correctly, as the dependencies were only added if the OPENMP_ENABLE_LIBOMPTARGET was set - but that variable is only set by the openmp/CMakeLists.txt file, which isn't even parsed during the initial cmake run (in fact, it is only parsed when executing the runtimes configure step itself, but then it is too late). Fixed by just adding those dependencies always. In addition, the list of dependencies collected in ${extra_deps}, including those required for OpenMP, was only actually used when configuring runtimes for the default set of targets - when the user specifies a non-default LLVM_RUNTIME_TARGETS, those extra dependencies were ignored (with the exception of ${hdrgen_deps}). Fixed by passing the full ${extra_deps} in this case as well. Fixes: https://github.com/llvm/llvm-project/issues/85933 --- llvm/runtimes/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt index 623c43d564cc..8159d7f8a0a1 100644 --- a/llvm/runtimes/CMakeLists.txt +++ b/llvm/runtimes/CMakeLists.txt @@ -435,7 +435,7 @@ if(runtimes) list(APPEND extra_deps "flang-new") endif() foreach(dep opt llvm-link llvm-extract clang clang-offload-packager) - if(TARGET ${dep} AND OPENMP_ENABLE_LIBOMPTARGET) + if(TARGET ${dep}) list(APPEND extra_deps ${dep}) endif() endforeach() @@ -531,7 +531,7 @@ if(runtimes) check_apple_target(${name} runtime) runtime_register_target(${name} - DEPENDS ${builtins_dep_name} ${hdrgen_deps} + DEPENDS ${builtins_dep_name} ${extra_deps} CMAKE_ARGS -DLLVM_DEFAULT_TARGET_TRIPLE=${name} ${libc_cmake_args} EXTRA_ARGS TARGET_TRIPLE ${name}) endforeach() -- GitLab From 8779edb8b33e8ae7e021c8fa7fff80d77567b28c Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Thu, 21 Mar 2024 12:14:24 +0100 Subject: [PATCH 029/933] [libc++] Deprecates std::errc constants. (#80542) Implements: - LWG3869 Deprecate std::errc constants related to UNIX STREAMS --- libcxx/docs/Status/Cxx23Issues.csv | 2 +- libcxx/include/__system_error/errc.h | 70 +++++++++++++++---- libcxx/include/cerrno | 13 ++++ libcxx/src/random.cpp | 4 +- .../test/std/depr.cerro/cerrno.syn.verify.cpp | 37 ++++++++++ .../depr.cerro/system.error.syn.verify.cpp | 28 ++++++++ .../test/std/diagnostics/syserr/errc.pass.cpp | 2 + 7 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp create mode 100644 libcxx/test/std/depr.cerro/system.error.syn.verify.cpp diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv index a10319235006..8de265f4d1b6 100644 --- a/libcxx/docs/Status/Cxx23Issues.csv +++ b/libcxx/docs/Status/Cxx23Issues.csv @@ -295,7 +295,7 @@ "`3847 `__","``ranges::to`` can still return views","February 2023","|Complete|","17.0","|ranges|" "`3862 `__","``basic_const_iterator``'s ``common_type`` specialization is underconstrained","February 2023","","","" "`3865 `__","Sorting a range of ``pairs``","February 2023","|Complete|","17.0","|ranges|" -"`3869 `__","Deprecate ``std::errc`` constants related to UNIX STREAMS","February 2023","","","" +"`3869 `__","Deprecate ``std::errc`` constants related to UNIX STREAMS","February 2023","|Complete|","19.0","" "`3870 `__","Remove ``voidify``","February 2023","","","" "`3871 `__","Adjust note about ``terminate``","February 2023","","","" "`3872 `__","``basic_const_iterator`` should have custom ``iter_move``","February 2023","","","" diff --git a/libcxx/include/__system_error/errc.h b/libcxx/include/__system_error/errc.h index f87df86a71e1..e9f3656b7b9c 100644 --- a/libcxx/include/__system_error/errc.h +++ b/libcxx/include/__system_error/errc.h @@ -58,18 +58,18 @@ enum class errc no_child_process, // ECHILD no_link, // ENOLINK no_lock_available, // ENOLCK - no_message_available, // ENODATA + no_message_available, // ENODATA // deprecated no_message, // ENOMSG no_protocol_option, // ENOPROTOOPT no_space_on_device, // ENOSPC - no_stream_resources, // ENOSR + no_stream_resources, // ENOSR // deprecated no_such_device_or_address, // ENXIO no_such_device, // ENODEV no_such_file_or_directory, // ENOENT no_such_process, // ESRCH not_a_directory, // ENOTDIR not_a_socket, // ENOTSOCK - not_a_stream, // ENOSTR + not_a_stream, // ENOSTR // deprecated not_connected, // ENOTCONN not_enough_memory, // ENOMEM not_supported, // ENOTSUP @@ -87,7 +87,7 @@ enum class errc resource_unavailable_try_again, // EAGAIN result_out_of_range, // ERANGE state_not_recoverable, // ENOTRECOVERABLE - stream_timeout, // ETIME + stream_timeout, // ETIME // deprecated text_file_busy, // ETXTBSY timed_out, // ETIMEDOUT too_many_files_open_in_system, // ENFILE @@ -107,12 +107,34 @@ enum class errc # pragma GCC system_header #endif +// The method of pushing and popping the diagnostics fails for GCC. GCC does +// not recognize the pragma's used to generate deprecated diagnostics for +// macros. So GCC does not need the pushing and popping. +// +// TODO Remove this when the deprecated constants are removed. +#if defined(_LIBCPP_COMPILER_CLANG_BASED) +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH _LIBCPP_SUPPRESS_DEPRECATED_PUSH +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP _LIBCPP_SUPPRESS_DEPRECATED_POP +#else +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP +#endif + _LIBCPP_BEGIN_NAMESPACE_STD // Some error codes are not present on all platforms, so we provide equivalents // for them: // enum class errc +// +// LWG3869 deprecates the UNIX STREAMS macros and enum values. +// This makes the code clumbersome: +// - the enum value is deprecated and should show a diagnostic, +// - the macro is deprecated and should _not_ show a diagnostic in this +// context, and +// - the macro is not always available. +// This leads to the odd pushing and popping of the deprecated +// diagnostic. _LIBCPP_DECLARE_STRONG_ENUM(errc){ address_family_not_supported = EAFNOSUPPORT, address_in_use = EADDRINUSE, @@ -154,30 +176,48 @@ _LIBCPP_DECLARE_STRONG_ENUM(errc){ no_child_process = ECHILD, no_link = ENOLINK, no_lock_available = ENOLCK, + // clang-format off + no_message_available _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENODATA - no_message_available = ENODATA, + ENODATA #else - no_message_available = ENOMSG, + ENOMSG #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on no_message = ENOMSG, no_protocol_option = ENOPROTOOPT, no_space_on_device = ENOSPC, + // clang-format off + no_stream_resources _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENOSR - no_stream_resources = ENOSR, + ENOSR #else - no_stream_resources = ENOMEM, + ENOMEM #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on no_such_device_or_address = ENXIO, no_such_device = ENODEV, no_such_file_or_directory = ENOENT, no_such_process = ESRCH, not_a_directory = ENOTDIR, not_a_socket = ENOTSOCK, + // clang-format off + not_a_stream _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENOSTR - not_a_stream = ENOSTR, + ENOSTR #else - not_a_stream = EINVAL, + EINVAL #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on not_connected = ENOTCONN, not_enough_memory = ENOMEM, not_supported = ENOTSUP, @@ -195,11 +235,17 @@ _LIBCPP_DECLARE_STRONG_ENUM(errc){ resource_unavailable_try_again = EAGAIN, result_out_of_range = ERANGE, state_not_recoverable = ENOTRECOVERABLE, + // clang-format off + stream_timeout _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ETIME - stream_timeout = ETIME, + ETIME #else - stream_timeout = ETIMEDOUT, + ETIMEDOUT #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on text_file_busy = ETXTBSY, timed_out = ETIMEDOUT, too_many_files_open_in_system = ENFILE, diff --git a/libcxx/include/cerrno b/libcxx/include/cerrno index d488fa72a54b..6171ae31f184 100644 --- a/libcxx/include/cerrno +++ b/libcxx/include/cerrno @@ -38,4 +38,17 @@ Macros: # pragma GCC system_header #endif +#ifdef ENODATA +# pragma clang deprecated(ENODATA, "ENODATA is deprecated in ISO C++") +#endif +#ifdef ENOSR +# pragma clang deprecated(ENOSR, "ENOSR is deprecated in ISO C++") +#endif +#ifdef ENOSTR +# pragma clang deprecated(ENOSTR, "ENOSTR is deprecated in ISO C++") +#endif +#ifdef ETIME +# pragma clang deprecated(ETIME, "ETIME is deprecated in ISO C++") +#endif + #endif // _LIBCPP_CERRNO diff --git a/libcxx/src/random.cpp b/libcxx/src/random.cpp index c7073c54da6b..93590af310e5 100644 --- a/libcxx/src/random.cpp +++ b/libcxx/src/random.cpp @@ -79,8 +79,10 @@ unsigned random_device::operator()() { char* p = reinterpret_cast(&r); while (n > 0) { ssize_t s = read(__f_, p, n); + _LIBCPP_SUPPRESS_DEPRECATED_PUSH if (s == 0) - __throw_system_error(ENODATA, "random_device got EOF"); + __throw_system_error(ENODATA, "random_device got EOF"); // TODO ENODATA -> ENOMSG + _LIBCPP_SUPPRESS_DEPRECATED_POP if (s == -1) { if (errno != EINTR) __throw_system_error(errno, "random_device got an unexpected error"); diff --git a/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp b/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp new file mode 100644 index 000000000000..3a38605570da --- /dev/null +++ b/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: clang-modules-build +// UNSUPPORTED: apple-clang && c++03 + +// + +// tests LWG 3869 deprecated macros. +// +// Note the macros may not be defined. When they are not defined the +// ifdef XXX does not trigger a deprecated message. So use them in the +// ifdef and test for 2 deprecated messages. + +#include + +#ifdef ENODATA +[[maybe_unused]] int nodata = + ENODATA; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENODATA' has been marked as deprecated}} +#endif +#ifdef ENOSR +[[maybe_unused]] int nosr = + ENOSR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSR' has been marked as deprecated}} +#endif +#ifdef ENOSTR +[[maybe_unused]] int nostr = + ENOSTR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSTR' has been marked as deprecated}} +#endif +#ifdef ETIME +[[maybe_unused]] int timeout = + ETIME; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ETIME' has been marked as deprecated}} +#endif diff --git a/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp b/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp new file mode 100644 index 000000000000..fab5dd5b5593 --- /dev/null +++ b/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// These macros do not seem to behave as expected on all Apple platforms. +// Since the macros are not provided newer POSIX versions it is expected the +// macros will be retroactively removed from C++. (The deprecation was +// retroactively.) +// UNSUPPORTED: apple-clang && (c++03 || clang-modules-build) + +// + +// enum errc {...} + +// tests LWG 3869 deprecated enum members. + +#include + +[[maybe_unused]] std::errc nodata = + std::errc::no_message_available; // expected-warning {{'no_message_available' is deprecated}} +[[maybe_unused]] std::errc nosr = + std::errc::no_stream_resources; // expected-warning {{'no_stream_resources' is deprecated}} +[[maybe_unused]] std::errc nostr = std::errc::not_a_stream; // expected-warning {{'not_a_stream' is deprecated}} +[[maybe_unused]] std::errc timeout = std::errc::stream_timeout; // expected-warning {{'stream_timeout' is deprecated}} diff --git a/libcxx/test/std/diagnostics/syserr/errc.pass.cpp b/libcxx/test/std/diagnostics/syserr/errc.pass.cpp index e44cb50102e3..4abee08ddc66 100644 --- a/libcxx/test/std/diagnostics/syserr/errc.pass.cpp +++ b/libcxx/test/std/diagnostics/syserr/errc.pass.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS + // // enum errc {...} -- GitLab From 7e72cafd68335e45d95ea6b7705bb5b9e7e442c8 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:15:17 -0400 Subject: [PATCH 030/933] [SelectionDAG] Add MaskedValueIsZero check to allow folding of zero extended variables we know are safe to extend (#85573) Add ones for every high bit that will cleared. This will allow us to evaluate variables that have their bits known to see if they have no risk of overflow despite the shift amount being greater than the difference between the two types. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 ++- llvm/test/CodeGen/X86/dagcombine-shifts.ll | 127 ++++++++++++++++++ 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index a3f5d433d920..c83793d15b2e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -13832,11 +13832,20 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::SHL) { // If the original shl may be shifting out bits, do not perform this // transformation. - // TODO: Add MaskedValueIsZero check. unsigned KnownZeroBits = ShVal.getValueSizeInBits() - ShVal.getOperand(0).getValueSizeInBits(); - if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) - return SDValue(); + if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) { + // If the shift is too large, then see if we can deduce that the + // shift is safe anyway. + // Create a mask that has ones for the bits being shifted out. + APInt ShiftOutMask = + APInt::getHighBitsSet(ShVal.getValueSizeInBits(), + ShAmtC->getAPIntValue().getZExtValue()); + + // Check if the bits being shifted out are known to be zero. + if (!DAG.MaskedValueIsZero(ShVal, ShiftOutMask)) + return SDValue(); + } } // Ensure that the shift amount is wide enough for the shifted value. diff --git a/llvm/test/CodeGen/X86/dagcombine-shifts.ll b/llvm/test/CodeGen/X86/dagcombine-shifts.ll index 42b325dd4c22..734abfe55a4e 100644 --- a/llvm/test/CodeGen/X86/dagcombine-shifts.ll +++ b/llvm/test/CodeGen/X86/dagcombine-shifts.ll @@ -322,5 +322,132 @@ define void @g(i32 %a) nounwind { ret void } +define i32 @shift_zext_shl(i8 zeroext %x) { +; X86-LABEL: shift_zext_shl: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; X86-NEXT: andl $64, %eax +; X86-NEXT: shll $9, %eax +; X86-NEXT: retl +; +; X64-LABEL: shift_zext_shl: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $64, %eax +; X64-NEXT: shll $9, %eax +; X64-NEXT: retq + %a = and i8 %x, 64 + %b = zext i8 %a to i16 + %c = shl i16 %b, 9 + %d = zext i16 %c to i32 + ret i32 %d +} + +define i32 @shift_zext_shl2(i8 zeroext %x) { +; X86-LABEL: shift_zext_shl2: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; X86-NEXT: andl $64, %eax +; X86-NEXT: shll $9, %eax +; X86-NEXT: retl +; +; X64-LABEL: shift_zext_shl2: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $64, %eax +; X64-NEXT: shll $9, %eax +; X64-NEXT: retq + %a = and i8 %x, 64 + %b = zext i8 %a to i32 + %c = shl i32 %b, 9 + ret i32 %c +} + +define <4 x i32> @shift_zext_shl_vec(<4 x i8> %x) nounwind { +; X86-LABEL: shift_zext_shl_vec: +; X86: # %bb.0: +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: andl $64, %ecx +; X86-NEXT: shll $9, %ecx +; X86-NEXT: andl $63, %edx +; X86-NEXT: shll $8, %edx +; X86-NEXT: andl $31, %esi +; X86-NEXT: shll $7, %esi +; X86-NEXT: andl $23, %edi +; X86-NEXT: shll $6, %edi +; X86-NEXT: movl %edi, 12(%eax) +; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 +; +; X64-LABEL: shift_zext_shl_vec: +; X64: # %bb.0: +; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; X64-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X64-NEXT: retq + %a = and <4 x i8> %x, + %b = zext <4 x i8> %a to <4 x i16> + %c = shl <4 x i16> %b, + %d = zext <4 x i16> %c to <4 x i32> + ret <4 x i32> %d +} + +define <4 x i32> @shift_zext_shl2_vec(<4 x i8> %x) nounwind { +; X86-LABEL: shift_zext_shl2_vec: +; X86: # %bb.0: +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edi +; X86-NEXT: andl $23, %edi +; X86-NEXT: andl $31, %esi +; X86-NEXT: andl $63, %edx +; X86-NEXT: andl $64, %ecx +; X86-NEXT: shll $9, %ecx +; X86-NEXT: shll $8, %edx +; X86-NEXT: shll $7, %esi +; X86-NEXT: shll $6, %edi +; X86-NEXT: movl %edi, 12(%eax) +; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 +; +; X64-LABEL: shift_zext_shl2_vec: +; X64: # %bb.0: +; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X64-NEXT: retq + %a = and <4 x i8> %x, + %b = zext <4 x i8> %a to <4 x i32> + %c = shl <4 x i32> %b, + ret <4 x i32> %c +} + declare dso_local void @f(i64) -- GitLab From 83e5a1239242d64110e3dfa96ed3889170ab96b2 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 12:21:44 +0100 Subject: [PATCH 031/933] [mlir][bazel] Don't expose interface headers from //mlir:IR. (#85867) Move 3 interface headers in `//mlir:IR` from `hdrs` to `srcs`. Header files should not be added to multiple targets, but this is hard to avoid because CMake is less strict with headers. But we should at least avoid exposing them as headers by multiple targets because it confuses tooling. --- .../llvm-project-overlay/mlir/BUILD.bazel | 22 ++++++++++++++++--- .../mlir/examples/toy/Ch5/BUILD.bazel | 1 + .../mlir/examples/toy/Ch6/BUILD.bazel | 1 + .../mlir/examples/toy/Ch7/BUILD.bazel | 1 + .../mlir/test/BUILD.bazel | 4 ++++ .../mlir/unittests/BUILD.bazel | 1 + 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 3951a31bae3e..e20aebe95063 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -330,16 +330,16 @@ cc_library( "lib/Bytecode/*.h", ]) + [ "include/mlir/IR/PDLPatternMatch.h.inc", + "include/mlir/Interfaces/CallInterfaces.h", + "include/mlir/Interfaces/DataLayoutInterfaces.h", + "include/mlir/Interfaces/SideEffectInterfaces.h", "lib/Bytecode/BytecodeOpInterface.cpp", ], hdrs = glob([ "include/mlir/IR/*.h", "include/mlir/Bytecode/*.h", ]) + [ - "include/mlir/Interfaces/CallInterfaces.h", - "include/mlir/Interfaces/DataLayoutInterfaces.h", "include/mlir/Interfaces/FoldInterfaces.h", - "include/mlir/Interfaces/SideEffectInterfaces.h", ], includes = ["include"], deps = [ @@ -1646,6 +1646,7 @@ cc_library( ":IR", ":MemRefDialect", ":Pass", + ":SideEffectInterfaces", ":Support", ":TransformUtils", ":Transforms", @@ -3131,6 +3132,7 @@ cc_library( ":DialectUtils", ":IR", ":InferTypeOpInterface", + ":SideEffectInterfaces", ":SparseTensorAttrDefsIncGen", ":SparseTensorEnums", ":SparseTensorInterfacesIncGen", @@ -4017,6 +4019,7 @@ cc_library( ":AffineDialect", ":Analysis", ":ArithDialect", + ":CallOpInterfaces", ":DialectUtils", ":FuncDialect", ":IR", @@ -4100,6 +4103,7 @@ cc_library( ":Pass", ":SCFDialect", ":SCFUtils", + ":SideEffectInterfaces", ":Support", ":TensorDialect", ":Transforms", @@ -6947,6 +6951,7 @@ cc_library( ]), includes = ["include"], deps = [ + ":CallOpInterfaces", ":CommonFolders", ":ControlFlowInterfaces", ":FunctionInterfaces", @@ -7591,6 +7596,7 @@ cc_library( includes = ["include"], deps = [ ":Analysis", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -7921,6 +7927,7 @@ cc_library( includes = ["include"], deps = [ ":Analysis", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -7930,6 +7937,7 @@ cc_library( ":Pass", ":Rewrite", ":RuntimeVerifiableOpInterface", + ":SideEffectInterfaces", ":Support", ":TransformUtils", ":TransformsPassIncGen", @@ -8081,6 +8089,7 @@ cc_library( hdrs = glob(["include/mlir/Conversion/LLVMCommon/*.h"]), includes = ["include"], deps = [ + ":DataLayoutInterfaces", ":IR", ":LLVMDialect", ":Support", @@ -9052,6 +9061,7 @@ cc_library( includes = ["include"], deps = [ ":DLTIDialect", + ":DataLayoutInterfaces", ":IR", ":LLVMConversionIncGen", ":LLVMDialect", @@ -12681,6 +12691,7 @@ cc_library( ":ArithOpsIncGen", ":ArithOpsInterfacesIncGen", ":BufferizationInterfaces", + ":CallOpInterfaces", ":CastInterfaces", ":CommonFolders", ":ControlFlowInterfaces", @@ -12692,6 +12703,7 @@ cc_library( ":InferTypeOpInterface", ":InliningUtils", ":Pass", + ":SideEffectInterfaces", ":Support", ":UBDialect", ":ValueBoundsOpInterfaceIncGen", @@ -12998,6 +13010,7 @@ cc_library( ":ArithDialect", ":ArithUtils", ":BufferizationInterfaces", + ":CallOpInterfaces", ":CastInterfaces", ":ComplexDialect", ":ControlFlowInterfaces", @@ -13012,6 +13025,7 @@ cc_library( ":MemorySlotInterfaces", ":RuntimeVerifiableOpInterface", ":ShapedOpInterfaces", + ":SideEffectInterfaces", ":Support", ":ValueBoundsOpInterface", ":ViewLikeInterface", @@ -13259,6 +13273,7 @@ cc_library( hdrs = glob(["include/mlir/Dialect/MLProgram/IR/*.h"]), includes = ["include"], deps = [ + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -13267,6 +13282,7 @@ cc_library( ":MLProgramOpsIncGen", ":MLProgramTypesIncGen", ":Pass", + ":SideEffectInterfaces", ":Support", ":Transforms", "//llvm:Support", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel index ce48e249489d..2c49d52f1ed0 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel @@ -102,6 +102,7 @@ cc_binary( "//mlir:Analysis", "//mlir:ArithDialect", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:FuncDialect", "//mlir:FuncExtensions", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel index 286c08065645..cd7f7f018166 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel @@ -108,6 +108,7 @@ cc_binary( "//mlir:ArithToLLVM", "//mlir:BuiltinToLLVMIRTranslation", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:ControlFlowToLLVM", "//mlir:ExecutionEngine", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel index f4037cab03f6..c03672eb4136 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel @@ -108,6 +108,7 @@ cc_binary( "//mlir:ArithToLLVM", "//mlir:BuiltinToLLVMIRTranslation", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:ControlFlowToLLVM", "//mlir:ExecutionEngine", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index 771cbcc4eea0..df2392fd8c6e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -94,12 +94,14 @@ cc_library( "//mlir:AffineAnalysis", "//mlir:AffineDialect", "//mlir:Analysis", + "//mlir:CallOpInterfaces", "//mlir:ControlFlowInterfaces", "//mlir:FuncDialect", "//mlir:FunctionInterfaces", "//mlir:IR", "//mlir:MemRefDialect", "//mlir:Pass", + "//mlir:SideEffectInterfaces", "//mlir:Support", ], ) @@ -386,6 +388,7 @@ cc_library( ":TestTypeDefsIncGen", "//llvm:Support", "//mlir:ArithDialect", + "//mlir:CallOpInterfaces", "//mlir:ControlFlowInterfaces", "//mlir:CopyOpInterface", "//mlir:DLTIDialect", @@ -571,6 +574,7 @@ cc_library( "//mlir:Pass", "//mlir:SCFDialect", "//mlir:SPIRVDialect", + "//mlir:SideEffectInterfaces", "//mlir:Support", "//mlir:Transforms", ], diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index 252b9ec951f6..c6b630230bb3 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -40,6 +40,7 @@ cc_test( deps = [ "//llvm:Support", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:FunctionInterfaces", "//mlir:IR", "//mlir:Parser", -- GitLab From 3ac243bc0d7922d083af2cf025247b5698556062 Mon Sep 17 00:00:00 2001 From: SahilPatidar Date: Thu, 21 Mar 2024 16:52:08 +0530 Subject: [PATCH 032/933] Update amdgpu_gfx functions to use s0-s3 for inreg SGPR arguments on targets using scratch instructions for stack #78226 (#81394) Resolve #78226 --- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 5 +- llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +- .../GlobalISel/irtranslator-call-non-fixed.ll | 10 +- .../AMDGPU/GlobalISel/irtranslator-call.ll | 10 +- llvm/test/CodeGen/AMDGPU/bf16.ll | 2 +- .../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 24 +- .../CodeGen/AMDGPU/function-args-inreg.ll | 133 + .../AMDGPU/gfx-callable-argument-types.ll | 4371 +++++++---------- llvm/test/CodeGen/AMDGPU/indirect-call.ll | 60 +- .../CodeGen/AMDGPU/schedule-addrspaces.ll | 2 +- .../CodeGen/AMDGPU/scratch-pointer-sink.ll | 4 +- 12 files changed, 2069 insertions(+), 2560 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp index 7e1f041fa109..8969b3b5b6ce 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp @@ -715,10 +715,13 @@ bool AMDGPUCallLowering::lowerFormalArguments( if (!IsEntryFunc && !IsGraphics) { // For the fixed ABI, pass workitem IDs in the last argument register. TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); + } + if (!IsEntryFunc) { if (!Subtarget.enableFlatScratch()) CCInfo.AllocateReg(Info->getScratchRSrcReg()); - TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); + if (!IsGraphics) + TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); } IncomingValueAssigner Assigner(AssignFn); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td index 4be64629ddac..9bd0fd7eca76 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td @@ -23,6 +23,7 @@ def CC_SI_Gfx : CallingConv<[ // 33 is reserved for the frame pointer // 34 is reserved for the base pointer CCIfInRegenableFlatScratch()) CCInfo.AllocateReg(Info->getScratchRSrcReg()); - - allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); + if (!IsGraphics) + allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); } if (!IsKernel) { diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll index 5effd24a7520..fad833c0a6ad 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll @@ -50,10 +50,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg - ; CHECK-NEXT: $sgpr4 = COPY [[C]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[C]](s32) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42) @@ -99,11 +99,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() # ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16) - ; CHECK-NEXT: $sgpr4 = COPY [[ANYEXT1]](s32) - ; CHECK-NEXT: $sgpr5 = COPY [[LOAD2]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT1]](s32) + ; CHECK-NEXT: $sgpr1 = COPY [[LOAD2]](s32) ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll index 392b0ae6823e..75670604baa1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll @@ -942,10 +942,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg - ; CHECK-NEXT: $sgpr4 = COPY [[C]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[C]](s32) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42) @@ -3984,11 +3984,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() # ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16) - ; CHECK-NEXT: $sgpr4 = COPY [[ANYEXT1]](s32) - ; CHECK-NEXT: $sgpr5 = COPY [[LOAD2]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT1]](s32) + ; CHECK-NEXT: $sgpr1 = COPY [[LOAD2]](s32) ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 98658834e897..e369f7e3b9a5 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -3337,7 +3337,7 @@ define amdgpu_gfx void @test_inreg_arg_store(bfloat inreg %in, ptr addrspace(1) ; GFX11-LABEL: test_inreg_arg_store: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-NEXT: global_store_b16 v[0:1], v2, off ; GFX11-NEXT: s_setpc_b64 s[30:31] store bfloat %in, ptr addrspace(1) %out diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll index 10d71a315fbf..e1e3220cc275 100644 --- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll +++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll @@ -472,7 +472,7 @@ define amdgpu_gfx void @test34(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test34: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_i32 s0, s4, s5 +; GCN-NEXT: s_min_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_lt_i32 s0, 0x3e9 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -492,7 +492,7 @@ define amdgpu_gfx void @test35(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test35: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e8 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -512,9 +512,9 @@ define amdgpu_gfx void @test36(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test36: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_u32 s0, s4, s5 +; GCN-NEXT: s_min_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_lt_u32 s0, s6 +; GCN-NEXT: s_cmp_lt_u32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -532,9 +532,9 @@ define amdgpu_gfx void @test37(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test37: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_ge_i32 s0, s6 +; GCN-NEXT: s_cmp_ge_i32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -552,7 +552,7 @@ define amdgpu_gfx void @test38(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test38: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_u32 s0, s4, s5 +; GCN-NEXT: s_max_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_lt_u32 s0, 0x3e9 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -572,7 +572,7 @@ define amdgpu_gfx void @test39(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test39: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_i32 s0, s4, s5 +; GCN-NEXT: s_min_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e7 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -592,9 +592,9 @@ define amdgpu_gfx void @test40(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test40: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_le_i32 s0, s6 +; GCN-NEXT: s_cmp_le_i32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -612,9 +612,9 @@ define amdgpu_gfx void @test41(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test41: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_u32 s0, s4, s5 +; GCN-NEXT: s_min_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_ge_u32 s0, s6 +; GCN-NEXT: s_cmp_ge_u32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 diff --git a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll index 44a9127b4bd0..27845b6b5b2f 100644 --- a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll +++ b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll @@ -2176,6 +2176,93 @@ define void @void_func_a5i32_inreg([5 x i32] inreg %arg0, ptr addrspace(1) %ptr) declare void @extern() define void @void_func_a13i32_inreg([13 x i32] inreg %arg0, ptr addrspace(1) %ptr) { +; GFX9-LABEL: void_func_a13i32_inreg: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s27, s33 +; GFX9-NEXT: s_mov_b32 s33, s32 +; GFX9-NEXT: s_or_saveexec_b64 s[28:29], -1 +; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[28:29] +; GFX9-NEXT: v_mov_b32_e32 v2, s26 +; GFX9-NEXT: global_store_dword v[0:1], v2, off offset:48 +; GFX9-NEXT: v_mov_b32_e32 v5, s25 +; GFX9-NEXT: v_mov_b32_e32 v4, s24 +; GFX9-NEXT: v_mov_b32_e32 v3, s23 +; GFX9-NEXT: v_mov_b32_e32 v2, s22 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:32 +; GFX9-NEXT: v_writelane_b32 v40, s27, 2 +; GFX9-NEXT: v_mov_b32_e32 v5, s21 +; GFX9-NEXT: v_mov_b32_e32 v4, s20 +; GFX9-NEXT: v_mov_b32_e32 v3, s19 +; GFX9-NEXT: v_mov_b32_e32 v2, s18 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s17 +; GFX9-NEXT: v_mov_b32_e32 v4, s16 +; GFX9-NEXT: s_getpc_b64 s[16:17] +; GFX9-NEXT: s_add_u32 s16, s16, extern@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s17, s17, extern@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0 +; GFX9-NEXT: v_mov_b32_e32 v3, s7 +; GFX9-NEXT: v_mov_b32_e32 v2, s6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s4, v40, 2 +; GFX9-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[6:7] +; GFX9-NEXT: s_addk_i32 s32, 0xfc00 +; GFX9-NEXT: s_mov_b32 s33, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: void_func_a13i32_inreg: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_mov_b32 s23, s33 +; GFX11-NEXT: s_mov_b32 s33, s32 +; GFX11-NEXT: s_or_saveexec_b32 s24, -1 +; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill +; GFX11-NEXT: s_mov_b32 exec_lo, s24 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v3, s19 +; GFX11-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v9, s17 +; GFX11-NEXT: s_getpc_b64 s[18:19] +; GFX11-NEXT: s_add_u32 s18, s18, extern@gotpcrel32@lo+4 +; GFX11-NEXT: s_addc_u32 s19, s19, extern@gotpcrel32@hi+12 +; GFX11-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v7, s7 +; GFX11-NEXT: s_load_b64 s[16:17], s[18:19], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s23, 2 +; GFX11-NEXT: v_dual_mov_b32 v14, s22 :: v_dual_mov_b32 v5, s21 +; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v13, s3 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1 +; GFX11-NEXT: v_mov_b32_e32 v10, s0 +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: global_store_b32 v[0:1], v14, off offset:48 +; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off offset:32 +; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 +; GFX11-NEXT: global_store_b128 v[0:1], v[10:13], off +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 +; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload +; GFX11-NEXT: s_mov_b32 exec_lo, s1 +; GFX11-NEXT: s_add_i32 s32, s32, -16 +; GFX11-NEXT: s_mov_b32 s33, s0 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] store [13 x i32] %arg0, ptr addrspace(1) %ptr call void @extern() ret void @@ -2203,6 +2290,52 @@ define void @void_func_a13i32_inreg([13 x i32] inreg %arg0, ptr addrspace(1) %p ; FIXME: Should still fail define void @void_func_a16i32_inreg__noimplicit([16 x i32] inreg %arg0, ptr addrspace(1) %ptr) { +; GFX9-LABEL: void_func_a16i32_inreg__noimplicit: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v5, s19 +; GFX9-NEXT: v_mov_b32_e32 v4, s18 +; GFX9-NEXT: v_mov_b32_e32 v3, s17 +; GFX9-NEXT: v_mov_b32_e32 v2, s16 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:48 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s15 +; GFX9-NEXT: v_mov_b32_e32 v4, s14 +; GFX9-NEXT: v_mov_b32_e32 v3, s13 +; GFX9-NEXT: v_mov_b32_e32 v2, s12 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:32 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-NEXT: v_mov_b32_e32 v4, s10 +; GFX9-NEXT: v_mov_b32_e32 v3, s9 +; GFX9-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s7 +; GFX9-NEXT: v_mov_b32_e32 v4, s6 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: void_func_a16i32_inreg__noimplicit: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v5, s15 :: v_dual_mov_b32 v4, s14 +; GFX11-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v2, s12 +; GFX11-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10 +; GFX11-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v6, s8 +; GFX11-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6 +; GFX11-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4 +; GFX11-NEXT: v_dual_mov_b32 v17, s3 :: v_dual_mov_b32 v16, s2 +; GFX11-NEXT: v_dual_mov_b32 v15, s1 :: v_dual_mov_b32 v14, s0 +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off offset:48 +; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:32 +; GFX11-NEXT: global_store_b128 v[0:1], v[10:13], off offset:16 +; GFX11-NEXT: global_store_b128 v[0:1], v[14:17], off +; GFX11-NEXT: s_setpc_b64 s[30:31] store [16 x i32] %arg0, ptr addrspace(1) %ptr ret void } diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll index a118fa388f86..3e1db5fb4e1d 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll @@ -9567,19 +9567,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b +; GFX9-NEXT: s_movk_i32 s0, 0x7b ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9597,19 +9595,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9627,20 +9623,18 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9658,19 +9652,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9692,19 +9684,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b +; GFX9-NEXT: s_movk_i32 s0, 0x7b ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9722,19 +9712,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9752,20 +9740,18 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9783,19 +9769,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9817,19 +9801,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 42 +; GFX9-NEXT: s_mov_b32 s0, 42 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9847,19 +9829,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 42 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 42 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9877,20 +9857,18 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 42 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 42 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9908,19 +9886,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 42 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 42 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9942,22 +9918,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b -; GFX9-NEXT: s_mov_b32 s5, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7b +; GFX9-NEXT: s_mov_b32 s1, 0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9975,22 +9947,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b +; GFX10-NEXT: s_mov_b32 s1, 0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10008,23 +9976,19 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b +; GFX11-NEXT: s_mov_b32 s1, 0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10042,22 +10006,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10079,26 +10039,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 ; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10116,26 +10073,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10153,27 +10107,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10191,26 +10137,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10233,28 +10171,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10272,28 +10202,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10311,29 +10233,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10351,28 +10265,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10394,32 +10300,29 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 8 +; GFX9-NEXT: v_writelane_b32 v40, s34, 4 +; GFX9-NEXT: s_mov_b64 s[34:35], 0 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s30, 6 +; GFX9-NEXT: v_writelane_b32 v40, s30, 2 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s8, 1 -; GFX9-NEXT: s_mov_b32 s9, 2 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, 1 +; GFX9-NEXT: s_mov_b32 s5, 2 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 7 +; GFX9-NEXT: v_writelane_b32 v40, s31, 3 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 7 -; GFX9-NEXT: v_readlane_b32 s30, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 3 +; GFX9-NEXT: v_readlane_b32 s30, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 8 +; GFX9-NEXT: v_readlane_b32 s34, v40, 4 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10437,32 +10340,29 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 8 +; GFX10-NEXT: v_writelane_b32 v40, s34, 4 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 1 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 2 -; GFX10-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-NEXT: v_writelane_b32 v40, s31, 7 +; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 8 +; GFX10-NEXT: v_readlane_b32 s34, v40, 4 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10480,33 +10380,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 8 +; GFX11-NEXT: v_writelane_b32 v40, s0, 4 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 1 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 2 -; GFX11-NEXT: v_writelane_b32 v40, s30, 6 -; GFX11-NEXT: v_writelane_b32 v40, s31, 7 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s5, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 2 +; GFX11-NEXT: v_writelane_b32 v40, s31, 3 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 7 -; GFX11-NEXT: v_readlane_b32 s30, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 +; GFX11-NEXT: v_readlane_b32 s31, v40, 3 +; GFX11-NEXT: v_readlane_b32 s30, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 8 +; GFX11-NEXT: v_readlane_b32 s0, v40, 4 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10524,32 +10416,24 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 8 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 7 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 8 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10574,38 +10458,35 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: s_mov_b64 s[34:35], 0 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s8, 1 -; GFX9-NEXT: s_mov_b32 s9, 2 -; GFX9-NEXT: s_mov_b32 s10, 3 -; GFX9-NEXT: s_mov_b32 s11, 4 -; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, 1 +; GFX9-NEXT: s_mov_b32 s5, 2 +; GFX9-NEXT: s_mov_b32 s6, 3 +; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10623,38 +10504,35 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s4, 1 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, 2 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-NEXT: s_mov_b32 s6, 3 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 1 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 2 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: s_mov_b32 s10, 3 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-NEXT: s_mov_b32 s11, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: s_mov_b32 s7, 4 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10672,39 +10550,31 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 +; GFX11-NEXT: s_mov_b32 s5, 2 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 +; GFX11-NEXT: s_mov_b32 s6, 3 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 1 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 2 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: s_mov_b32 s10, 3 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 -; GFX11-NEXT: s_mov_b32 s11, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s7, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10722,38 +10592,30 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s10, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s11, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10777,19 +10639,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x4400 +; GFX9-NEXT: s_movk_i32 s0, 0x4400 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10807,19 +10667,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x4400 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x4400 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10837,20 +10695,18 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x4400 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x4400 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10868,19 +10724,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x4400 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x4400 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10902,19 +10756,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 4.0 +; GFX9-NEXT: s_mov_b32 s0, 4.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10932,19 +10784,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 4.0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10962,20 +10812,18 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 4.0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10993,19 +10841,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 4.0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11027,22 +10873,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11060,22 +10902,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11093,23 +10931,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11127,22 +10961,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11164,25 +10994,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 5 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 3 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 4.0 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 4.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 4 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 4 -; GFX9-NEXT: v_readlane_b32 s30, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 5 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11200,25 +11024,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 5 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 4.0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-NEXT: v_writelane_b32 v40, s31, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 5 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11236,26 +11054,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 5 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 4.0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 3 -; GFX11-NEXT: v_writelane_b32 v40, s31, 4 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 4 -; GFX11-NEXT: v_readlane_b32 s30, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 5 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11273,25 +11085,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 4.0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 4 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 5 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11313,31 +11119,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 7 +; GFX9-NEXT: v_writelane_b32 v40, s34, 3 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s30, 5 +; GFX9-NEXT: v_writelane_b32 v40, s30, 1 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 4.0 -; GFX9-NEXT: s_mov_b32 s7, -1.0 -; GFX9-NEXT: s_mov_b32 s8, 0.5 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 4.0 +; GFX9-NEXT: s_mov_b32 s3, -1.0 +; GFX9-NEXT: s_mov_b32 s4, 0.5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 2 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 6 -; GFX9-NEXT: v_readlane_b32 s30, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s31, v40, 2 +; GFX9-NEXT: v_readlane_b32 s30, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 7 +; GFX9-NEXT: v_readlane_b32 s34, v40, 3 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11355,31 +11153,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 7 +; GFX10-NEXT: v_writelane_b32 v40, s34, 3 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, -1.0 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 0.5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-NEXT: v_writelane_b32 v40, s31, 6 +; GFX10-NEXT: s_mov_b32 s2, 4.0 +; GFX10-NEXT: s_mov_b32 s3, -1.0 +; GFX10-NEXT: s_mov_b32 s4, 0.5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-NEXT: v_writelane_b32 v40, s31, 2 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 7 +; GFX10-NEXT: v_readlane_b32 s34, v40, 3 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11397,32 +11187,24 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 7 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, -1.0 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 0.5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 5 -; GFX11-NEXT: v_writelane_b32 v40, s31, 6 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s2, 4.0 +; GFX11-NEXT: s_mov_b32 s3, -1.0 +; GFX11-NEXT: s_mov_b32 s4, 0.5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s30, 1 +; GFX11-NEXT: v_writelane_b32 v40, s31, 2 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 6 -; GFX11-NEXT: v_readlane_b32 s30, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 +; GFX11-NEXT: v_readlane_b32 s31, v40, 2 +; GFX11-NEXT: v_readlane_b32 s30, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 7 +; GFX11-NEXT: v_readlane_b32 s0, v40, 3 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11440,31 +11222,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, -1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 0.5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 6 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 4.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, -1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0.5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 7 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11486,22 +11260,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 0x40100000 +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 0x40100000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11519,22 +11289,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: s_mov_b32 s1, 0x40100000 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11552,23 +11318,19 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 0x40100000 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11586,22 +11348,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0x40100000 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11623,28 +11381,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 0 -; GFX9-NEXT: s_mov_b32 s7, 0x40100000 +; GFX9-NEXT: s_mov_b32 s2, 0 +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s3, 0x40100000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11662,28 +11412,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11701,29 +11443,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 0 +; GFX11-NEXT: s_mov_b32 s3, 0x40100000 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11741,28 +11475,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11784,34 +11510,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 8 +; GFX9-NEXT: v_writelane_b32 v40, s34, 4 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s30, 6 +; GFX9-NEXT: v_writelane_b32 v40, s30, 2 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 0 +; GFX9-NEXT: s_mov_b32 s3, 0x40100000 ; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 0 -; GFX9-NEXT: s_mov_b32 s7, 0x40100000 -; GFX9-NEXT: s_mov_b32 s8, 0 -; GFX9-NEXT: s_mov_b32 s9, 0x40200000 +; GFX9-NEXT: s_mov_b32 s5, 0x40200000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 7 +; GFX9-NEXT: v_writelane_b32 v40, s31, 3 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 7 -; GFX9-NEXT: v_readlane_b32 s30, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 3 +; GFX9-NEXT: v_readlane_b32 s30, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 8 +; GFX9-NEXT: v_readlane_b32 s34, v40, 4 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11829,34 +11547,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 8 +; GFX10-NEXT: v_writelane_b32 v40, s34, 4 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-NEXT: s_mov_b32 s4, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 0 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 0x40200000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-NEXT: v_writelane_b32 v40, s31, 7 +; GFX10-NEXT: s_mov_b32 s5, 0x40200000 +; GFX10-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-NEXT: v_writelane_b32 v40, s31, 3 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 8 +; GFX10-NEXT: v_readlane_b32 s34, v40, 4 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11874,35 +11584,27 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 8 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s2, 0 +; GFX11-NEXT: s_mov_b32 s3, 0x40100000 ; GFX11-NEXT: s_mov_b32 s4, 0 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 0 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 0x40200000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 6 -; GFX11-NEXT: v_writelane_b32 v40, s31, 7 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s5, 0x40200000 +; GFX11-NEXT: v_writelane_b32 v40, s30, 2 +; GFX11-NEXT: v_writelane_b32 v40, s31, 3 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 7 -; GFX11-NEXT: v_readlane_b32 s30, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 +; GFX11-NEXT: v_readlane_b32 s31, v40, 3 +; GFX11-NEXT: v_readlane_b32 s30, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 8 +; GFX11-NEXT: v_readlane_b32 s0, v40, 4 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11920,34 +11622,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 8 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 0x40200000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 7 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40200000 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 8 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11969,19 +11663,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: s_load_dword s4, s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11999,19 +11691,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_load_dword s4, s[34:35], 0x0 +; GFX10-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12029,20 +11719,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12060,19 +11748,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dword s0, s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_load_dword s4, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12095,21 +11781,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12127,21 +11812,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12159,22 +11843,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12192,21 +11872,17 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12229,21 +11905,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12261,21 +11936,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12293,22 +11967,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12326,21 +11996,17 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12363,22 +12029,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x20001 -; GFX9-NEXT: s_mov_b32 s5, 3 +; GFX9-NEXT: s_mov_b32 s0, 0x20001 +; GFX9-NEXT: s_mov_b32 s1, 3 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12396,22 +12058,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-NEXT: s_mov_b32 s1, 3 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 3 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12429,23 +12087,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x20001 +; GFX11-NEXT: s_mov_b32 s1, 3 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x20001 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 3 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12463,22 +12117,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12500,22 +12150,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX9-NEXT: s_movk_i32 s5, 0x4400 +; GFX9-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX9-NEXT: s_movk_i32 s1, 0x4400 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12533,22 +12179,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX10-NEXT: s_movk_i32 s1, 0x4400 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_movk_i32 s5, 0x4400 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12566,23 +12208,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX11-NEXT: s_movk_i32 s1, 0x4400 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_movk_i32 s5, 0x4400 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12600,22 +12238,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX10-SCRATCH-NEXT: s_movk_i32 s1, 0x4400 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_movk_i32 s5, 0x4400 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12637,21 +12271,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12669,21 +12302,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12701,22 +12333,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12734,21 +12362,17 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12771,22 +12395,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x20001 -; GFX9-NEXT: s_mov_b32 s5, 0x40003 +; GFX9-NEXT: s_mov_b32 s0, 0x20001 +; GFX9-NEXT: s_mov_b32 s1, 0x40003 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12804,22 +12424,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-NEXT: s_mov_b32 s1, 0x40003 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0x40003 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12837,23 +12453,19 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x20001 +; GFX11-NEXT: s_mov_b32 s1, 0x40003 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x20001 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0x40003 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12871,22 +12483,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0x40003 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40003 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12908,19 +12516,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: s_load_dword s4, s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12938,19 +12544,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_load_dword s4, s[34:35], 0x0 +; GFX10-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12968,20 +12572,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12999,19 +12601,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dword s0, s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_load_dword s4, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13034,21 +12634,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13066,21 +12665,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13098,22 +12696,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13131,21 +12725,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13168,22 +12758,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13201,22 +12787,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13234,23 +12816,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13268,22 +12846,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13305,25 +12879,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 5 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 3 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 3 -; GFX9-NEXT: s_mov_b32 s5, 4 -; GFX9-NEXT: s_mov_b32 s6, 5 +; GFX9-NEXT: s_mov_b32 s0, 3 +; GFX9-NEXT: s_mov_b32 s1, 4 +; GFX9-NEXT: s_mov_b32 s2, 5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 4 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 4 -; GFX9-NEXT: v_readlane_b32 s30, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 5 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13341,25 +12909,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 5 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 3 +; GFX10-NEXT: s_mov_b32 s1, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 5 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 3 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 4 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-NEXT: v_writelane_b32 v40, s31, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 5 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13377,26 +12939,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 5 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 3 +; GFX11-NEXT: s_mov_b32 s1, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 5 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 3 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 4 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 3 -; GFX11-NEXT: v_writelane_b32 v40, s31, 4 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 4 -; GFX11-NEXT: v_readlane_b32 s30, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 5 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13414,25 +12970,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 5 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 4 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 5 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13454,28 +13004,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 3 -; GFX9-NEXT: s_mov_b32 s5, 4 -; GFX9-NEXT: s_mov_b32 s6, 5 -; GFX9-NEXT: s_mov_b32 s7, 6 +; GFX9-NEXT: s_mov_b32 s0, 3 +; GFX9-NEXT: s_mov_b32 s1, 4 +; GFX9-NEXT: s_mov_b32 s2, 5 +; GFX9-NEXT: s_mov_b32 s3, 6 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13493,28 +13035,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 3 +; GFX10-NEXT: s_mov_b32 s1, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 5 +; GFX10-NEXT: s_mov_b32 s3, 6 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 3 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 4 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 5 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 6 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13532,29 +13066,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 3 +; GFX11-NEXT: s_mov_b32 s1, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 5 +; GFX11-NEXT: s_mov_b32 s3, 6 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 3 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 4 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 5 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 6 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13572,28 +13098,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 5 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 6 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13615,25 +13133,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13651,25 +13166,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13687,26 +13199,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13724,25 +13228,17 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13765,28 +13261,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13804,28 +13292,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13843,29 +13323,21 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13883,28 +13355,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13926,31 +13390,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 7 +; GFX9-NEXT: v_writelane_b32 v40, s34, 3 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s30, 5 +; GFX9-NEXT: v_writelane_b32 v40, s30, 1 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 -; GFX9-NEXT: s_mov_b32 s8, 5 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 +; GFX9-NEXT: s_mov_b32 s4, 5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 2 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 6 -; GFX9-NEXT: v_readlane_b32 s30, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s31, v40, 2 +; GFX9-NEXT: v_readlane_b32 s30, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 7 +; GFX9-NEXT: v_readlane_b32 s34, v40, 3 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13965,34 +13421,26 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX10-NEXT: s_mov_b32 s34, s33 ; GFX10-NEXT: s_mov_b32 s33, s32 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 -; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 7 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-NEXT: v_writelane_b32 v40, s31, 6 +; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 +; GFX10-NEXT: s_mov_b32 s4, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-NEXT: v_writelane_b32 v40, s31, 2 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 7 +; GFX10-NEXT: v_readlane_b32 s34, v40, 3 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14010,32 +13458,24 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 7 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 5 -; GFX11-NEXT: v_writelane_b32 v40, s31, 6 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 +; GFX11-NEXT: s_mov_b32 s4, 5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s30, 1 +; GFX11-NEXT: v_writelane_b32 v40, s31, 2 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 6 -; GFX11-NEXT: v_readlane_b32 s30, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 +; GFX11-NEXT: v_readlane_b32 s31, v40, 2 +; GFX11-NEXT: v_readlane_b32 s30, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 7 +; GFX11-NEXT: v_readlane_b32 s0, v40, 3 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14053,31 +13493,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 6 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 7 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14099,35 +13531,36 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx8 s[4:11], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: s_load_dwordx8 s[36:43], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14145,35 +13578,36 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dwordx8 s[4:11], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx8 s[36:43], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14191,36 +13625,28 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b256 s[4:11], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_load_b256 s[0:7], s[0:1], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14238,35 +13664,27 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 ; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 ; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14290,40 +13708,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 -; GFX9-NEXT: s_mov_b32 s8, 5 -; GFX9-NEXT: s_mov_b32 s9, 6 -; GFX9-NEXT: s_mov_b32 s10, 7 -; GFX9-NEXT: s_mov_b32 s11, 8 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 +; GFX9-NEXT: s_mov_b32 s4, 5 +; GFX9-NEXT: s_mov_b32 s5, 6 +; GFX9-NEXT: s_mov_b32 s6, 7 +; GFX9-NEXT: s_mov_b32 s7, 8 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14341,40 +13751,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 +; GFX10-NEXT: s_mov_b32 s4, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 +; GFX10-NEXT: s_mov_b32 s5, 6 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 +; GFX10-NEXT: s_mov_b32 s6, 7 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 5 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 6 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: s_mov_b32 s10, 7 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-NEXT: s_mov_b32 s11, 8 -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: s_mov_b32 s7, 8 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14392,41 +13794,33 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 +; GFX11-NEXT: s_mov_b32 s4, 5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 +; GFX11-NEXT: s_mov_b32 s5, 6 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 +; GFX11-NEXT: s_mov_b32 s6, 7 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 5 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 6 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: s_mov_b32 s10, 7 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 -; GFX11-NEXT: s_mov_b32 s11, 8 -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s7, 8 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14444,40 +13838,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 6 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 7 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s10, 7 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s11, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 8 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14499,38 +13885,47 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 18 +; GFX9-NEXT: v_writelane_b32 v40, s34, 14 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 ; GFX9-NEXT: v_writelane_b32 v40, s8, 4 ; GFX9-NEXT: v_writelane_b32 v40, s9, 5 ; GFX9-NEXT: v_writelane_b32 v40, s10, 6 ; GFX9-NEXT: v_writelane_b32 v40, s11, 7 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s12, 8 ; GFX9-NEXT: v_writelane_b32 v40, s13, 9 ; GFX9-NEXT: v_writelane_b32 v40, s14, 10 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s15, 11 -; GFX9-NEXT: v_writelane_b32 v40, s16, 12 -; GFX9-NEXT: v_writelane_b32 v40, s17, 13 -; GFX9-NEXT: v_writelane_b32 v40, s18, 14 -; GFX9-NEXT: v_writelane_b32 v40, s19, 15 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 16 +; GFX9-NEXT: v_writelane_b32 v40, s30, 12 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 17 +; GFX9-NEXT: v_writelane_b32 v40, s31, 13 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 17 -; GFX9-NEXT: v_readlane_b32 s30, v40, 16 -; GFX9-NEXT: v_readlane_b32 s19, v40, 15 -; GFX9-NEXT: v_readlane_b32 s18, v40, 14 -; GFX9-NEXT: v_readlane_b32 s17, v40, 13 -; GFX9-NEXT: v_readlane_b32 s16, v40, 12 +; GFX9-NEXT: v_readlane_b32 s31, v40, 13 +; GFX9-NEXT: v_readlane_b32 s30, v40, 12 ; GFX9-NEXT: v_readlane_b32 s15, v40, 11 ; GFX9-NEXT: v_readlane_b32 s14, v40, 10 ; GFX9-NEXT: v_readlane_b32 s13, v40, 9 @@ -14543,7 +13938,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 18 +; GFX9-NEXT: v_readlane_b32 s34, v40, 14 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14561,38 +13956,47 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 18 +; GFX10-NEXT: v_writelane_b32 v40, s34, 14 ; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s8, 4 ; GFX10-NEXT: v_writelane_b32 v40, s9, 5 ; GFX10-NEXT: v_writelane_b32 v40, s10, 6 ; GFX10-NEXT: v_writelane_b32 v40, s11, 7 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: v_writelane_b32 v40, s12, 8 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 ; GFX10-NEXT: v_writelane_b32 v40, s13, 9 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 ; GFX10-NEXT: v_writelane_b32 v40, s14, 10 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 ; GFX10-NEXT: v_writelane_b32 v40, s15, 11 -; GFX10-NEXT: v_writelane_b32 v40, s16, 12 -; GFX10-NEXT: v_writelane_b32 v40, s17, 13 -; GFX10-NEXT: v_writelane_b32 v40, s18, 14 -; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 16 -; GFX10-NEXT: v_writelane_b32 v40, s31, 17 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: v_writelane_b32 v40, s30, 12 +; GFX10-NEXT: v_writelane_b32 v40, s31, 13 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 17 -; GFX10-NEXT: v_readlane_b32 s30, v40, 16 -; GFX10-NEXT: v_readlane_b32 s19, v40, 15 -; GFX10-NEXT: v_readlane_b32 s18, v40, 14 -; GFX10-NEXT: v_readlane_b32 s17, v40, 13 -; GFX10-NEXT: v_readlane_b32 s16, v40, 12 +; GFX10-NEXT: v_readlane_b32 s31, v40, 13 +; GFX10-NEXT: v_readlane_b32 s30, v40, 12 ; GFX10-NEXT: v_readlane_b32 s15, v40, 11 ; GFX10-NEXT: v_readlane_b32 s14, v40, 10 ; GFX10-NEXT: v_readlane_b32 s13, v40, 9 @@ -14605,7 +14009,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 18 +; GFX10-NEXT: v_readlane_b32 s34, v40, 14 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14623,8 +14027,10 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 18 +; GFX11-NEXT: v_writelane_b32 v40, s0, 14 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 @@ -14638,24 +14044,14 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s13, 9 ; GFX11-NEXT: v_writelane_b32 v40, s14, 10 ; GFX11-NEXT: v_writelane_b32 v40, s15, 11 -; GFX11-NEXT: v_writelane_b32 v40, s16, 12 -; GFX11-NEXT: v_writelane_b32 v40, s17, 13 -; GFX11-NEXT: v_writelane_b32 v40, s18, 14 -; GFX11-NEXT: v_writelane_b32 v40, s19, 15 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 16 -; GFX11-NEXT: v_writelane_b32 v40, s31, 17 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 12 +; GFX11-NEXT: v_writelane_b32 v40, s31, 13 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 17 -; GFX11-NEXT: v_readlane_b32 s30, v40, 16 -; GFX11-NEXT: v_readlane_b32 s19, v40, 15 -; GFX11-NEXT: v_readlane_b32 s18, v40, 14 -; GFX11-NEXT: v_readlane_b32 s17, v40, 13 -; GFX11-NEXT: v_readlane_b32 s16, v40, 12 +; GFX11-NEXT: v_readlane_b32 s31, v40, 13 +; GFX11-NEXT: v_readlane_b32 s30, v40, 12 ; GFX11-NEXT: v_readlane_b32 s15, v40, 11 ; GFX11-NEXT: v_readlane_b32 s14, v40, 10 ; GFX11-NEXT: v_readlane_b32 s13, v40, 9 @@ -14668,7 +14064,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 18 +; GFX11-NEXT: v_readlane_b32 s0, v40, 14 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14686,8 +14082,10 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 18 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 14 ; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 @@ -14701,23 +14099,13 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s13, 9 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s14, 10 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s15, 11 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s16, 12 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 ; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 17 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 17 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 16 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s19, v40, 15 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s18, v40, 14 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s17, v40, 13 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s16, v40, 12 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 12 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 13 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 13 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 12 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s15, v40, 11 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s14, v40, 10 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s13, v40, 9 @@ -14730,7 +14118,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 18 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 14 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14771,49 +14159,47 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX9-NEXT: v_writelane_b32 v40, s17, 13 ; GFX9-NEXT: v_writelane_b32 v40, s18, 14 ; GFX9-NEXT: v_writelane_b32 v40, s19, 15 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s20, 16 ; GFX9-NEXT: v_writelane_b32 v40, s21, 17 ; GFX9-NEXT: v_writelane_b32 v40, s22, 18 ; GFX9-NEXT: v_writelane_b32 v40, s23, 19 ; GFX9-NEXT: v_writelane_b32 v40, s24, 20 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s25, 21 ; GFX9-NEXT: v_writelane_b32 v40, s26, 22 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s27, 23 -; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s28, 24 +; GFX9-NEXT: v_writelane_b32 v40, s29, 25 +; GFX9-NEXT: v_writelane_b32 v40, s30, 26 +; GFX9-NEXT: v_writelane_b32 v40, s31, 27 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: s_mov_b32 s53, external_void_func_v32i32_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s52, external_void_func_v32i32_inreg@abs32@lo ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s46 -; GFX9-NEXT: v_writelane_b32 v40, s29, 25 -; GFX9-NEXT: v_mov_b32_e32 v1, s47 -; GFX9-NEXT: v_mov_b32_e32 v2, s48 -; GFX9-NEXT: v_mov_b32_e32 v3, s49 +; GFX9-NEXT: v_mov_b32_e32 v0, s30 +; GFX9-NEXT: v_mov_b32_e32 v1, s31 ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX9-NEXT: v_mov_b32_e32 v0, s50 -; GFX9-NEXT: v_writelane_b32 v40, s30, 26 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 -; GFX9-NEXT: v_mov_b32_e32 v0, s51 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s20, s36 -; GFX9-NEXT: s_mov_b32 s21, s37 -; GFX9-NEXT: s_mov_b32 s22, s38 -; GFX9-NEXT: s_mov_b32 s23, s39 -; GFX9-NEXT: s_mov_b32 s24, s40 -; GFX9-NEXT: s_mov_b32 s25, s41 -; GFX9-NEXT: s_mov_b32 s26, s42 -; GFX9-NEXT: s_mov_b32 s27, s43 -; GFX9-NEXT: s_mov_b32 s28, s44 -; GFX9-NEXT: s_mov_b32 s29, s45 -; GFX9-NEXT: v_writelane_b32 v40, s31, 27 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[52:53] ; GFX9-NEXT: v_readlane_b32 s31, v40, 27 ; GFX9-NEXT: v_readlane_b32 s30, v40, 26 ; GFX9-NEXT: v_readlane_b32 s29, v40, 25 @@ -14879,47 +14265,46 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s46 ; GFX10-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-NEXT: v_mov_b32_e32 v2, s48 -; GFX10-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-NEXT: s_mov_b32 s20, s36 ; GFX10-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-NEXT: s_mov_b32 s21, s37 -; GFX10-NEXT: s_mov_b32 s22, s38 -; GFX10-NEXT: s_mov_b32 s23, s39 -; GFX10-NEXT: s_mov_b32 s24, s40 ; GFX10-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-NEXT: s_mov_b32 s25, s41 -; GFX10-NEXT: v_mov_b32_e32 v4, s50 -; GFX10-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; GFX10-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-NEXT: s_mov_b32 s26, s42 ; GFX10-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-NEXT: s_mov_b32 s27, s43 ; GFX10-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-NEXT: s_mov_b32 s28, s44 ; GFX10-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-NEXT: s_mov_b32 s29, s45 ; GFX10-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-NEXT: v_writelane_b32 v40, s31, 27 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-NEXT: v_readlane_b32 s30, v40, 26 @@ -14970,8 +14355,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s0, 28 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s2, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 @@ -14988,42 +14373,26 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s17, 13 ; GFX11-NEXT: v_writelane_b32 v40, s18, 14 ; GFX11-NEXT: v_writelane_b32 v40, s19, 15 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_clause 0x1 -; GFX11-NEXT: s_load_b512 s[36:51], s[0:1], 0x40 -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s20, 16 ; GFX11-NEXT: v_writelane_b32 v40, s21, 17 ; GFX11-NEXT: v_writelane_b32 v40, s22, 18 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v5, s51 ; GFX11-NEXT: v_writelane_b32 v40, s23, 19 -; GFX11-NEXT: v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v1, s47 -; GFX11-NEXT: v_dual_mov_b32 v2, s48 :: v_dual_mov_b32 v3, s49 ; GFX11-NEXT: v_writelane_b32 v40, s24, 20 -; GFX11-NEXT: s_mov_b32 s20, s36 -; GFX11-NEXT: s_mov_b32 s21, s37 -; GFX11-NEXT: s_mov_b32 s22, s38 -; GFX11-NEXT: s_mov_b32 s23, s39 ; GFX11-NEXT: v_writelane_b32 v40, s25, 21 -; GFX11-NEXT: s_mov_b32 s24, s40 -; GFX11-NEXT: s_mov_b32 s25, s41 -; GFX11-NEXT: scratch_store_b64 off, v[4:5], s2 -; GFX11-NEXT: scratch_store_b128 off, v[0:3], s32 ; GFX11-NEXT: v_writelane_b32 v40, s26, 22 -; GFX11-NEXT: s_mov_b32 s26, s42 ; GFX11-NEXT: v_writelane_b32 v40, s27, 23 -; GFX11-NEXT: s_mov_b32 s27, s43 ; GFX11-NEXT: v_writelane_b32 v40, s28, 24 -; GFX11-NEXT: s_mov_b32 s28, s44 ; GFX11-NEXT: v_writelane_b32 v40, s29, 25 -; GFX11-NEXT: s_mov_b32 s29, s45 ; GFX11-NEXT: v_writelane_b32 v40, s30, 26 ; GFX11-NEXT: v_writelane_b32 v40, s31, 27 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b512 s[16:31], s[0:1], 0x40 +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v0, s30 :: v_dual_mov_b32 v1, s31 +; GFX11-NEXT: scratch_store_b64 off, v[0:1], s32 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: v_readlane_b32 s31, v40, 27 ; GFX11-NEXT: v_readlane_b32 s30, v40, 26 ; GFX11-NEXT: v_readlane_b32 s29, v40, 25 @@ -15071,9 +14440,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 28 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 @@ -15090,44 +14458,29 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_clause 0x1 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v4, s50 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s48 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-SCRATCH-NEXT: s_mov_b32 s20, s36 -; GFX10-SCRATCH-NEXT: s_mov_b32 s21, s37 -; GFX10-SCRATCH-NEXT: s_mov_b32 s22, s38 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-SCRATCH-NEXT: s_mov_b32 s23, s39 -; GFX10-SCRATCH-NEXT: s_mov_b32 s24, s40 -; GFX10-SCRATCH-NEXT: s_mov_b32 s25, s41 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[4:5], s2 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-SCRATCH-NEXT: s_mov_b32 s26, s42 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-SCRATCH-NEXT: s_mov_b32 s27, s43 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-SCRATCH-NEXT: s_mov_b32 s28, s44 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-SCRATCH-NEXT: s_mov_b32 s29, s45 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 27 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[0:1], s32 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 26 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s29, v40, 25 @@ -15196,55 +14549,53 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX9-NEXT: v_writelane_b32 v40, s16, 12 ; GFX9-NEXT: v_writelane_b32 v40, s17, 13 ; GFX9-NEXT: v_writelane_b32 v40, s18, 14 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s19, 15 ; GFX9-NEXT: v_writelane_b32 v40, s20, 16 ; GFX9-NEXT: v_writelane_b32 v40, s21, 17 ; GFX9-NEXT: v_writelane_b32 v40, s22, 18 ; GFX9-NEXT: v_writelane_b32 v40, s23, 19 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dword s52, s[34:35], 0x0 -; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 -; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 -; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s24, 20 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s25, 21 -; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s26, 22 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s52 ; GFX9-NEXT: v_writelane_b32 v40, s27, 23 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 -; GFX9-NEXT: v_mov_b32_e32 v0, s46 ; GFX9-NEXT: v_writelane_b32 v40, s28, 24 -; GFX9-NEXT: v_mov_b32_e32 v1, s47 -; GFX9-NEXT: v_mov_b32_e32 v2, s48 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX9-NEXT: v_mov_b32_e32 v0, s49 ; GFX9-NEXT: v_writelane_b32 v40, s29, 25 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 -; GFX9-NEXT: v_mov_b32_e32 v0, s50 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dword s52, s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s30, 26 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 -; GFX9-NEXT: v_mov_b32_e32 v0, s51 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s20, s36 -; GFX9-NEXT: s_mov_b32 s21, s37 -; GFX9-NEXT: s_mov_b32 s22, s38 -; GFX9-NEXT: s_mov_b32 s23, s39 -; GFX9-NEXT: s_mov_b32 s24, s40 -; GFX9-NEXT: s_mov_b32 s25, s41 -; GFX9-NEXT: s_mov_b32 s26, s42 -; GFX9-NEXT: s_mov_b32 s27, s43 -; GFX9-NEXT: s_mov_b32 s28, s44 -; GFX9-NEXT: s_mov_b32 s29, s45 ; GFX9-NEXT: v_writelane_b32 v40, s31, 27 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v0, s52 +; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, s30 +; GFX9-NEXT: s_mov_b32 s53, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX9-NEXT: v_mov_b32_e32 v1, s31 +; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX9-NEXT: s_mov_b32 s52, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 +; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 +; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[52:53] ; GFX9-NEXT: v_readlane_b32 s31, v40, 27 ; GFX9-NEXT: v_readlane_b32 s30, v40, 26 ; GFX9-NEXT: v_readlane_b32 s29, v40, 25 @@ -15310,52 +14661,51 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dword s52, s[34:35], 0x0 -; GFX10-NEXT: ; meta instruction -; GFX10-NEXT: ; meta instruction -; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s52 -; GFX10-NEXT: v_mov_b32_e32 v1, s47 ; GFX10-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 -; GFX10-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-NEXT: v_mov_b32_e32 v2, s48 -; GFX10-NEXT: v_mov_b32_e32 v3, s49 ; GFX10-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-NEXT: s_mov_b32 s20, s36 -; GFX10-NEXT: s_mov_b32 s21, s37 -; GFX10-NEXT: s_mov_b32 s22, s38 -; GFX10-NEXT: s_mov_b32 s23, s39 ; GFX10-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-NEXT: s_mov_b32 s24, s40 -; GFX10-NEXT: s_mov_b32 s25, s41 -; GFX10-NEXT: v_mov_b32_e32 v4, s50 -; GFX10-NEXT: v_mov_b32_e32 v5, s51 ; GFX10-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-NEXT: s_mov_b32 s26, s42 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; GFX10-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-NEXT: s_mov_b32 s27, s43 ; GFX10-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-NEXT: s_mov_b32 s28, s44 ; GFX10-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-NEXT: s_mov_b32 s29, s45 ; GFX10-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-NEXT: v_writelane_b32 v40, s31, 27 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: s_load_dword s52, s[34:35], 0x0 +; GFX10-NEXT: ; meta instruction +; GFX10-NEXT: ; meta instruction +; GFX10-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s52 +; GFX10-NEXT: v_mov_b32_e32 v1, s30 +; GFX10-NEXT: v_mov_b32_e32 v2, s31 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 +; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:4 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-NEXT: v_readlane_b32 s30, v40, 26 @@ -15406,8 +14756,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX11-NEXT: v_writelane_b32 v40, s0, 28 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s3, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX11-NEXT: s_add_i32 s36, s32, 8 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 @@ -15424,46 +14774,30 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX11-NEXT: v_writelane_b32 v40, s17, 13 ; GFX11-NEXT: v_writelane_b32 v40, s18, 14 ; GFX11-NEXT: v_writelane_b32 v40, s19, 15 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_clause 0x2 -; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0 -; GFX11-NEXT: s_load_b512 s[36:51], s[0:1], 0x40 -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s20, 16 ; GFX11-NEXT: v_writelane_b32 v40, s21, 17 ; GFX11-NEXT: v_writelane_b32 v40, s22, 18 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s51 ; GFX11-NEXT: v_writelane_b32 v40, s23, 19 -; GFX11-NEXT: v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v1, s47 -; GFX11-NEXT: v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v3, s49 ; GFX11-NEXT: v_writelane_b32 v40, s24, 20 -; GFX11-NEXT: v_mov_b32_e32 v2, s48 -; GFX11-NEXT: s_add_i32 s2, s32, 24 -; GFX11-NEXT: s_mov_b32 s20, s36 -; GFX11-NEXT: s_mov_b32 s21, s37 ; GFX11-NEXT: v_writelane_b32 v40, s25, 21 -; GFX11-NEXT: s_mov_b32 s22, s38 -; GFX11-NEXT: s_mov_b32 s23, s39 -; GFX11-NEXT: s_mov_b32 s24, s40 -; GFX11-NEXT: s_mov_b32 s25, s41 ; GFX11-NEXT: v_writelane_b32 v40, s26, 22 -; GFX11-NEXT: s_mov_b32 s26, s42 -; GFX11-NEXT: scratch_store_b32 off, v6, s2 -; GFX11-NEXT: scratch_store_b64 off, v[4:5], s3 -; GFX11-NEXT: scratch_store_b128 off, v[0:3], s32 ; GFX11-NEXT: v_writelane_b32 v40, s27, 23 -; GFX11-NEXT: s_mov_b32 s27, s43 ; GFX11-NEXT: v_writelane_b32 v40, s28, 24 -; GFX11-NEXT: s_mov_b32 s28, s44 ; GFX11-NEXT: v_writelane_b32 v40, s29, 25 -; GFX11-NEXT: s_mov_b32 s29, s45 ; GFX11-NEXT: v_writelane_b32 v40, s30, 26 ; GFX11-NEXT: v_writelane_b32 v40, s31, 27 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x2 +; GFX11-NEXT: s_load_b32 s34, s[0:1], 0x0 +; GFX11-NEXT: s_load_b512 s[16:31], s[0:1], 0x40 +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v2, s34 :: v_dual_mov_b32 v1, s31 +; GFX11-NEXT: v_mov_b32_e32 v0, s30 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX11-NEXT: scratch_store_b32 off, v2, s36 +; GFX11-NEXT: scratch_store_b64 off, v[0:1], s32 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: v_readlane_b32 s31, v40, 27 ; GFX11-NEXT: v_readlane_b32 s30, v40, 26 ; GFX11-NEXT: v_readlane_b32 s29, v40, 25 @@ -15511,13 +14845,17 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 28 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_load_dword s36, s[0:1], 0x0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: s_add_i32 s3, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s36 +; GFX10-SCRATCH-NEXT: s_add_i32 s36, s32, 8 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 @@ -15530,50 +14868,29 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_clause 0x2 -; GFX10-SCRATCH-NEXT: s_load_dword s2, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: ; meta instruction -; GFX10-SCRATCH-NEXT: ; meta instruction -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v6, s2 -; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 24 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v4, s50 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s48 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-SCRATCH-NEXT: s_mov_b32 s20, s36 -; GFX10-SCRATCH-NEXT: s_mov_b32 s21, s37 -; GFX10-SCRATCH-NEXT: s_mov_b32 s22, s38 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-SCRATCH-NEXT: s_mov_b32 s23, s39 -; GFX10-SCRATCH-NEXT: s_mov_b32 s24, s40 -; GFX10-SCRATCH-NEXT: s_mov_b32 s25, s41 -; GFX10-SCRATCH-NEXT: scratch_store_dword off, v6, s2 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[4:5], s3 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-SCRATCH-NEXT: s_mov_b32 s26, s42 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-SCRATCH-NEXT: s_mov_b32 s27, s43 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-SCRATCH-NEXT: s_mov_b32 s28, s44 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-SCRATCH-NEXT: s_mov_b32 s29, s45 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 27 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-SCRATCH-NEXT: scratch_store_dword off, v2, s36 +; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[0:1], s32 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 26 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s29, v40, 25 @@ -17397,6 +16714,7 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17423,6 +16741,7 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17442,18 +16761,18 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX11-LABEL: test_call_external_void_func_bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17469,19 +16788,19 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX10-SCRATCH-LABEL: test_call_external_void_func_bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17511,6 +16830,7 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v1bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v1bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17537,6 +16857,7 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v1bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v1bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17556,18 +16877,18 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX11-LABEL: test_call_external_void_func_v1bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v1bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v1bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v1bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v1bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17583,19 +16904,19 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v1bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v1bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v1bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v1bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v1bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17625,6 +16946,7 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17651,6 +16973,7 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17670,18 +16993,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX11-LABEL: test_call_external_void_func_v2bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17697,19 +17020,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v2bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17739,6 +17062,8 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17765,8 +17090,10 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -17784,18 +17111,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v3bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s2, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s3, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s3 +; GFX11-NEXT: v_writelane_b32 v40, s2, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17811,19 +17138,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v3bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s3, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s2, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17853,6 +17180,8 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17879,8 +17208,10 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -17898,18 +17229,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v4bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s2, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s3, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s3 +; GFX11-NEXT: v_writelane_b32 v40, s2, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17925,19 +17256,19 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v4bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s3, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s2, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17967,6 +17298,10 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s3, s7 +; GFX9-NEXT: s_mov_b32 s2, s6 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17993,8 +17328,12 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s3, s7 +; GFX10-NEXT: s_mov_b32 s2, s6 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -18012,18 +17351,18 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v8bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s34, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s35, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s35 +; GFX11-NEXT: v_writelane_b32 v40, s34, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -18039,19 +17378,19 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v8bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -18077,16 +17416,32 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: v_writelane_b32 v40, s4, 0 +; GFX9-NEXT: v_writelane_b32 v40, s5, 1 +; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: v_writelane_b32 v40, s7, 3 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s3, s7 +; GFX9-NEXT: s_mov_b32 s2, s6 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 +; GFX9-NEXT: s_mov_b32 s4, s8 +; GFX9-NEXT: s_mov_b32 s5, s9 +; GFX9-NEXT: s_mov_b32 s6, s10 +; GFX9-NEXT: s_mov_b32 s7, s11 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 1 -; GFX9-NEXT: v_readlane_b32 s30, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 +; GFX9-NEXT: v_readlane_b32 s7, v40, 3 +; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s4, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -18104,16 +17459,32 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s3, s7 +; GFX10-NEXT: s_mov_b32 s2, s6 +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 +; GFX10-NEXT: s_mov_b32 s4, s8 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s30, 0 -; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, s9 +; GFX10-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-NEXT: s_mov_b32 s6, s10 +; GFX10-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-NEXT: s_mov_b32 s7, s11 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 1 -; GFX10-NEXT: v_readlane_b32 s30, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 +; GFX10-NEXT: v_readlane_b32 s7, v40, 3 +; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s4, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -18126,18 +17497,18 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX11-LABEL: test_call_external_void_func_v16bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s34, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s35, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v16bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v16bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s35 +; GFX11-NEXT: v_writelane_b32 v40, s34, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -18153,19 +17524,19 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v16bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v16bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v16bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll index 7799b9509ceb..25c684004446 100644 --- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll @@ -847,11 +847,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-LABEL: test_indirect_call_vgpr_ptr_inreg_arg: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s5, s33 +; GCN-NEXT: s_mov_b32 s10, s33 ; GCN-NEXT: s_mov_b32 s33, s32 -; GCN-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_addk_i32 s32, 0x400 ; GCN-NEXT: v_writelane_b32 v40, s30, 0 ; GCN-NEXT: v_writelane_b32 v40, s31, 1 @@ -885,19 +885,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-NEXT: v_writelane_b32 v40, s61, 29 ; GCN-NEXT: v_writelane_b32 v40, s62, 30 ; GCN-NEXT: v_writelane_b32 v40, s63, 31 -; GCN-NEXT: s_mov_b64 s[6:7], exec -; GCN-NEXT: s_movk_i32 s4, 0x7b +; GCN-NEXT: s_mov_b64 s[4:5], exec ; GCN-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1 -; GCN-NEXT: v_readfirstlane_b32 s8, v0 -; GCN-NEXT: v_readfirstlane_b32 s9, v1 -; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1] -; GCN-NEXT: s_and_saveexec_b64 s[10:11], vcc -; GCN-NEXT: s_swappc_b64 s[30:31], s[8:9] +; GCN-NEXT: v_readfirstlane_b32 s6, v0 +; GCN-NEXT: v_readfirstlane_b32 s7, v1 +; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1] +; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GCN-NEXT: s_movk_i32 s0, 0x7b +; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1 -; GCN-NEXT: s_xor_b64 exec, exec, s[10:11] +; GCN-NEXT: s_xor_b64 exec, exec, s[8:9] ; GCN-NEXT: s_cbranch_execnz .LBB6_1 ; GCN-NEXT: ; %bb.2: -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: v_readlane_b32 s63, v40, 31 ; GCN-NEXT: v_readlane_b32 s62, v40, 30 ; GCN-NEXT: v_readlane_b32 s61, v40, 29 @@ -930,22 +930,22 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-NEXT: v_readlane_b32 s34, v40, 2 ; GCN-NEXT: v_readlane_b32 s31, v40, 1 ; GCN-NEXT: v_readlane_b32 s30, v40, 0 -; GCN-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_addk_i32 s32, 0xfc00 -; GCN-NEXT: s_mov_b32 s33, s5 +; GCN-NEXT: s_mov_b32 s33, s10 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GISEL-LABEL: test_indirect_call_vgpr_ptr_inreg_arg: ; GISEL: ; %bb.0: ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: s_mov_b32 s5, s33 +; GISEL-NEXT: s_mov_b32 s10, s33 ; GISEL-NEXT: s_mov_b32 s33, s32 -; GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GISEL-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: s_addk_i32 s32, 0x400 ; GISEL-NEXT: v_writelane_b32 v40, s30, 0 ; GISEL-NEXT: v_writelane_b32 v40, s31, 1 @@ -979,19 +979,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GISEL-NEXT: v_writelane_b32 v40, s61, 29 ; GISEL-NEXT: v_writelane_b32 v40, s62, 30 ; GISEL-NEXT: v_writelane_b32 v40, s63, 31 -; GISEL-NEXT: s_mov_b64 s[6:7], exec -; GISEL-NEXT: s_movk_i32 s4, 0x7b +; GISEL-NEXT: s_mov_b64 s[4:5], exec ; GISEL-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1 -; GISEL-NEXT: v_readfirstlane_b32 s8, v0 -; GISEL-NEXT: v_readfirstlane_b32 s9, v1 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1] -; GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc -; GISEL-NEXT: s_swappc_b64 s[30:31], s[8:9] +; GISEL-NEXT: v_readfirstlane_b32 s6, v0 +; GISEL-NEXT: v_readfirstlane_b32 s7, v1 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1] +; GISEL-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GISEL-NEXT: s_movk_i32 s0, 0x7b +; GISEL-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: s_xor_b64 exec, exec, s[10:11] +; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9] ; GISEL-NEXT: s_cbranch_execnz .LBB6_1 ; GISEL-NEXT: ; %bb.2: -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: v_readlane_b32 s63, v40, 31 ; GISEL-NEXT: v_readlane_b32 s62, v40, 30 ; GISEL-NEXT: v_readlane_b32 s61, v40, 29 @@ -1024,11 +1024,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GISEL-NEXT: v_readlane_b32 s34, v40, 2 ; GISEL-NEXT: v_readlane_b32 s31, v40, 1 ; GISEL-NEXT: v_readlane_b32 s30, v40, 0 -; GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GISEL-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: s_addk_i32 s32, 0xfc00 -; GISEL-NEXT: s_mov_b32 s33, s5 +; GISEL-NEXT: s_mov_b32 s33, s10 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_setpc_b64 s[30:31] call amdgpu_gfx void %fptr(i32 inreg 123) diff --git a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll index 0139c52db1d5..27e9d0fce942 100644 --- a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll +++ b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll @@ -10,7 +10,7 @@ define amdgpu_gfx void @example(<4 x i32> inreg %rsrc, ptr addrspace(5) %src, i3 ; CHECK-NEXT: scratch_load_b32 v2, v0, off ; CHECK-NEXT: scratch_load_b32 v3, v3, off ; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: buffer_store_b64 v[2:3], v1, s[4:7], 0 offen +; CHECK-NEXT: buffer_store_b64 v[2:3], v1, s[0:3], 0 offen ; CHECK-NEXT: s_setpc_b64 s[30:31] %x0 = load i32, ptr addrspace(5) %src diff --git a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll index cdaac14833e0..ec1de0222d5d 100644 --- a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll +++ b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll @@ -6,7 +6,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg % ; GCN-LABEL: sink_scratch_pointer: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_cmp_lg_u32 s4, 0 +; GCN-NEXT: s_cmp_lg_u32 s0, 0 ; GCN-NEXT: s_cbranch_scc0 .LBB0_2 ; GCN-NEXT: ; %bb.1: ; %bb2 ; GCN-NEXT: scratch_load_b32 v0, v0, off offset:-4 @@ -21,7 +21,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg % ; GISEL-LABEL: sink_scratch_pointer: ; GISEL: ; %bb.0: ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: s_cmp_lg_u32 s4, 0 +; GISEL-NEXT: s_cmp_lg_u32 s0, 0 ; GISEL-NEXT: s_cbranch_scc0 .LBB0_2 ; GISEL-NEXT: ; %bb.1: ; %bb2 ; GISEL-NEXT: scratch_load_b32 v0, v0, off offset:-4 -- GitLab From 34f0a8aaba11bf703ddd2de92eee8ecbb77be5c8 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 04:13:23 -0700 Subject: [PATCH 033/933] [SLP]Fix comparison in bitwidth check. Projected bitwidth should be less than the original, not greater. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 +- .../orig-btiwidth-les-projected.ll | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a52064e5417b..4853c2006fea 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -13995,7 +13995,7 @@ bool BoUpSLP::collectValuesToDemote( if (MultiNodeScalars.contains(V)) return false; uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); - if (OrigBitWidth < BitWidth) { + if (OrigBitWidth > BitWidth) { APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) return true; diff --git a/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll b/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll new file mode 100644 index 000000000000..531e96405348 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer < %s | FileCheck %s + +define i32 @test(i4 %0) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: i4 [[TMP0:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP1:%.*]] = trunc i8 0 to i4 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i8 0 to i4 +; CHECK-NEXT: [[ADD_R:%.*]] = or i4 [[TMP1]], [[TMP0]] +; CHECK-NEXT: [[ADD_R14:%.*]] = or i4 0, [[TMP2]] +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i4 [[ADD_R]], [[ADD_R14]] +; CHECK-NEXT: ret i32 0 +; +entry: + %1 = trunc i8 0 to i4 + %2 = trunc i8 0 to i4 + %add.r = or i4 %1, %0 + %add.r14 = or i4 0, %2 + %cmp.not = icmp eq i4 %add.r, %add.r14 + ret i32 0 +} -- GitLab From df6a1d44094e187d7d5ea3ee5b54b9bccc8a4798 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 12:32:12 +0100 Subject: [PATCH 034/933] [mlir][tensor] NFC: fully qualify verifyEncoding arguments. --- mlir/include/mlir/IR/TensorEncoding.td | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/IR/TensorEncoding.td b/mlir/include/mlir/IR/TensorEncoding.td index 3991520d72a5..4907dcbb5de9 100644 --- a/mlir/include/mlir/IR/TensorEncoding.td +++ b/mlir/include/mlir/IR/TensorEncoding.td @@ -34,8 +34,8 @@ def VerifiableTensorEncoding : AttrInterface<"VerifiableTensorEncoding"> { /*retTy=*/"::mlir::LogicalResult", /*methodName=*/"verifyEncoding", /*args=*/(ins - "ArrayRef":$shape, - "Type":$elementType, + "::mlir::ArrayRef":$shape, + "::mlir::Type":$elementType, "::llvm::function_ref<::mlir::InFlightDiagnostic()>":$emitError) >, ]; -- GitLab From 2699072b4bc8d8d5e84eb66af38face73ceeb4d3 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Thu, 21 Mar 2024 12:57:24 +0100 Subject: [PATCH 035/933] [clang] Accept lambdas in C++03 as an extensions (#73376) Implements https://discourse.llvm.org/t/rfc-allow-c-11-lambdas-in-c-03-as-an-extension/75262 --- clang/docs/LanguageExtensions.rst | 73 ++++++----- clang/docs/ReleaseNotes.rst | 2 + .../clang/Basic/DiagnosticParseKinds.td | 1 + clang/include/clang/Basic/Features.def | 1 + clang/lib/Parse/ParseExpr.cpp | 2 +- clang/lib/Parse/ParseExprCXX.cpp | 9 +- clang/lib/Parse/ParseInit.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 2 +- clang/lib/Sema/SemaDeclCXX.cpp | 3 +- clang/test/Lexer/has_extension_cxx.cpp | 5 + .../OpenMP/declare_reduction_messages.cpp | 14 +-- clang/test/OpenMP/openmp_check.cpp | 2 +- clang/test/Parser/cxx03-lambda-extension.cpp | 5 + .../test/Parser/cxx0x-lambda-expressions.cpp | 116 +++++++----------- clang/test/Parser/cxx2b-lambdas.cpp | 45 +++++-- .../Parser/objcxx-lambda-expressions-neg.mm | 9 +- clang/test/ParserHLSL/group_shared.hlsl | 4 +- clang/test/SemaCXX/cxx2a-template-lambdas.cpp | 26 ++-- clang/test/SemaCXX/lambda-expressions.cpp | 64 ++++++---- .../SemaCXX/lambda-implicit-this-capture.cpp | 1 + clang/test/SemaCXX/lambda-invalid-capture.cpp | 1 + clang/test/SemaCXX/new-delete.cpp | 7 +- 22 files changed, 213 insertions(+), 181 deletions(-) create mode 100644 clang/test/Parser/cxx03-lambda-extension.cpp diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 13d7261d83d7..201a4c27f7dd 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -1459,40 +1459,45 @@ More information could be found `here Language Extensions Back-ported to Previous Standards ===================================================== -====================================== ================================ ============= ============= -Feature Feature Test Macro Introduced In Backported To -====================================== ================================ ============= ============= -variadic templates __cpp_variadic_templates C++11 C++03 -Alias templates __cpp_alias_templates C++11 C++03 -Non-static data member initializers __cpp_nsdmi C++11 C++03 -Range-based ``for`` loop __cpp_range_based_for C++11 C++03 -RValue references __cpp_rvalue_references C++11 C++03 -Attributes __cpp_attributes C++11 C++03 -variable templates __cpp_variable_templates C++14 C++03 -Binary literals __cpp_binary_literals C++14 C++03 -Relaxed constexpr __cpp_constexpr C++14 C++11 -``if constexpr`` __cpp_if_constexpr C++17 C++11 -fold expressions __cpp_fold_expressions C++17 C++03 -Lambda capture of \*this by value __cpp_capture_star_this C++17 C++11 -Attributes on enums __cpp_enumerator_attributes C++17 C++03 -Guaranteed copy elision __cpp_guaranteed_copy_elision C++17 C++03 -Hexadecimal floating literals __cpp_hex_float C++17 C++03 -``inline`` variables __cpp_inline_variables C++17 C++03 -Attributes on namespaces __cpp_namespace_attributes C++17 C++11 -Structured bindings __cpp_structured_bindings C++17 C++03 -template template arguments __cpp_template_template_args C++17 C++03 -``static operator[]`` __cpp_multidimensional_subscript C++20 C++03 -Designated initializers __cpp_designated_initializers C++20 C++03 -Conditional ``explicit`` __cpp_conditional_explicit C++20 C++03 -``using enum`` __cpp_using_enum C++20 C++03 -``if consteval`` __cpp_if_consteval C++23 C++20 -``static operator()`` __cpp_static_call_operator C++23 C++03 -Attributes on Lambda-Expressions C++23 C++11 --------------------------------------- -------------------------------- ------------- ------------- -Designated initializers (N494) C99 C89 -Array & element qualification (N2607) C23 C89 -Attributes (N2335) C23 C89 -====================================== ================================ ============= ============= +============================================ ================================ ============= ============= +Feature Feature Test Macro Introduced In Backported To +============================================ ================================ ============= ============= +variadic templates __cpp_variadic_templates C++11 C++03 +Alias templates __cpp_alias_templates C++11 C++03 +Non-static data member initializers __cpp_nsdmi C++11 C++03 +Range-based ``for`` loop __cpp_range_based_for C++11 C++03 +RValue references __cpp_rvalue_references C++11 C++03 +Attributes __cpp_attributes C++11 C++03 +Lambdas __cpp_lambdas C++11 C++03 +Generalized lambda captures __cpp_init_captures C++14 C++03 +Generic lambda expressions __cpp_generic_lambdas C++14 C++03 +variable templates __cpp_variable_templates C++14 C++03 +Binary literals __cpp_binary_literals C++14 C++03 +Relaxed constexpr __cpp_constexpr C++14 C++11 +Pack expansion in generalized lambda-capture __cpp_init_captures C++17 C++03 +``if constexpr`` __cpp_if_constexpr C++17 C++11 +fold expressions __cpp_fold_expressions C++17 C++03 +Lambda capture of \*this by value __cpp_capture_star_this C++17 C++03 +Attributes on enums __cpp_enumerator_attributes C++17 C++03 +Guaranteed copy elision __cpp_guaranteed_copy_elision C++17 C++03 +Hexadecimal floating literals __cpp_hex_float C++17 C++03 +``inline`` variables __cpp_inline_variables C++17 C++03 +Attributes on namespaces __cpp_namespace_attributes C++17 C++11 +Structured bindings __cpp_structured_bindings C++17 C++03 +template template arguments __cpp_template_template_args C++17 C++03 +Familiar template syntax for generic lambdas __cpp_generic_lambdas C++20 C++03 +``static operator[]`` __cpp_multidimensional_subscript C++20 C++03 +Designated initializers __cpp_designated_initializers C++20 C++03 +Conditional ``explicit`` __cpp_conditional_explicit C++20 C++03 +``using enum`` __cpp_using_enum C++20 C++03 +``if consteval`` __cpp_if_consteval C++23 C++20 +``static operator()`` __cpp_static_call_operator C++23 C++03 +Attributes on Lambda-Expressions C++23 C++11 +-------------------------------------------- -------------------------------- ------------- ------------- +Designated initializers (N494) C99 C89 +Array & element qualification (N2607) C23 C89 +Attributes (N2335) C23 C89 +============================================ ================================ ============= ============= Type Trait Primitives ===================== diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index a9c55ef662a0..50990140a53a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -180,6 +180,8 @@ Non-comprehensive list of changes in this release the previous builtins, this new builtin is constexpr and may be used in constant expressions. +- Lambda expressions are now accepted in C++03 mode as an extension. + New Compiler Flags ------------------ diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index 816c3ff5f8b2..48de5e2ef5f4 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1029,6 +1029,7 @@ def err_expected_lambda_body : Error<"expected body of lambda expression">; def warn_cxx98_compat_lambda : Warning< "lambda expressions are incompatible with C++98">, InGroup, DefaultIgnore; +def ext_lambda : ExtWarn<"lambdas are a C++11 extension">, InGroup; def err_lambda_decl_specifier_repeated : Error< "%select{'mutable'|'static'|'constexpr'|'consteval'}0 cannot " "appear multiple times in a lambda declarator">; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 726ead4b5ab5..b41aadc73f20 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -261,6 +261,7 @@ EXTENSION(cxx_defaulted_functions, LangOpts.CPlusPlus) EXTENSION(cxx_deleted_functions, LangOpts.CPlusPlus) EXTENSION(cxx_explicit_conversions, LangOpts.CPlusPlus) EXTENSION(cxx_inline_namespaces, LangOpts.CPlusPlus) +EXTENSION(cxx_lambdas, LangOpts.CPlusPlus) EXTENSION(cxx_local_type_template_args, LangOpts.CPlusPlus) EXTENSION(cxx_nonstatic_member_init, LangOpts.CPlusPlus) EXTENSION(cxx_override_control, LangOpts.CPlusPlus) diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 88c3a1469e8e..ae23cb432c43 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -1823,7 +1823,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, } goto ExpectedExpression; case tok::l_square: - if (getLangOpts().CPlusPlus11) { + if (getLangOpts().CPlusPlus) { if (getLangOpts().ObjC) { // C++11 lambda expressions and Objective-C message sends both start with a // square bracket. There are three possibilities here: diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 9471f6f725ef..73c85c585baa 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -806,9 +806,8 @@ ExprResult Parser::ParseLambdaExpression() { /// /// If we are not looking at a lambda expression, returns ExprError(). ExprResult Parser::TryParseLambdaExpression() { - assert(getLangOpts().CPlusPlus11 - && Tok.is(tok::l_square) - && "Not at the start of a possible lambda expression."); + assert(getLangOpts().CPlusPlus && Tok.is(tok::l_square) && + "Not at the start of a possible lambda expression."); const Token Next = NextToken(); if (Next.is(tok::eof)) // Nothing else to lookup here... @@ -1326,7 +1325,9 @@ static void DiagnoseStaticSpecifierRestrictions(Parser &P, ExprResult Parser::ParseLambdaExpressionAfterIntroducer( LambdaIntroducer &Intro) { SourceLocation LambdaBeginLoc = Intro.Range.getBegin(); - Diag(LambdaBeginLoc, diag::warn_cxx98_compat_lambda); + Diag(LambdaBeginLoc, getLangOpts().CPlusPlus11 + ? diag::warn_cxx98_compat_lambda + : diag::ext_lambda); PrettyStackTraceLoc CrashInfo(PP.getSourceManager(), LambdaBeginLoc, "lambda expression parsing"); diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp index 637f21176792..423497bfcb66 100644 --- a/clang/lib/Parse/ParseInit.cpp +++ b/clang/lib/Parse/ParseInit.cpp @@ -35,7 +35,7 @@ bool Parser::MayBeDesignationStart() { return true; case tok::l_square: { // designator: array-designator - if (!PP.getLangOpts().CPlusPlus11) + if (!PP.getLangOpts().CPlusPlus) return true; // C++11 lambda expressions and C99 designators can be ambiguous all the diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 8ceb79555fb5..aa754d47a0c4 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -16109,7 +16109,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, FD->setInvalidDecl(); } } - } else if (getLangOpts().CPlusPlus11 && isLambdaCallOperator(FD)) { + } else if (getLangOpts().CPlusPlus && isLambdaCallOperator(FD)) { // In C++11, we don't use 'auto' deduction rules for lambda call // operators because we don't support return type deduction. auto *LSI = getCurLambda(); diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index e258a4f7c894..ee732679417e 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -9738,7 +9738,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM, return false; CXXRecordDecl *RD = MD->getParent(); assert(!RD->isDependentType() && "do deletion after instantiation"); - if (!LangOpts.CPlusPlus11 || RD->isInvalidDecl()) + if (!LangOpts.CPlusPlus || (!LangOpts.CPlusPlus11 && !RD->isLambda()) || + RD->isInvalidDecl()) return false; // C++11 [expr.lambda.prim]p19: diff --git a/clang/test/Lexer/has_extension_cxx.cpp b/clang/test/Lexer/has_extension_cxx.cpp index 7941997428ac..7366029d3727 100644 --- a/clang/test/Lexer/has_extension_cxx.cpp +++ b/clang/test/Lexer/has_extension_cxx.cpp @@ -33,6 +33,11 @@ int has_deleted_functions(); int has_inline_namespaces(); #endif +// CHECK: has_lambdas +#if __has_extension(cxx_lambdas) +int has_lambdas(); +#endif + // CHECK: has_override_control #if __has_extension(cxx_override_control) int has_override_control(); diff --git a/clang/test/OpenMP/declare_reduction_messages.cpp b/clang/test/OpenMP/declare_reduction_messages.cpp index 38a5d766eead..752cc4fb05a1 100644 --- a/clang/test/OpenMP/declare_reduction_messages.cpp +++ b/clang/test/OpenMP/declare_reduction_messages.cpp @@ -58,16 +58,10 @@ class Class2 : public Class1 { #pragma omp declare reduction(fun1 : long : omp_out += omp_in) initializer // expected-error {{expected '(' after 'initializer'}} #pragma omp declare reduction(fun2 : long : omp_out += omp_in) initializer { // expected-error {{expected '(' after 'initializer'}} expected-error {{expected expression}} expected-warning {{extra tokens at the end of '#pragma omp declare reduction' are ignored}} #pragma omp declare reduction(fun3 : long : omp_out += omp_in) initializer[ -#if __cplusplus <= 199711L -// expected-error@-2 {{expected '(' after 'initializer'}} -// expected-error@-3 {{expected expression}} -// expected-warning@-4 {{extra tokens at the end of '#pragma omp declare reduction' are ignored}} -#else -// expected-error@-6 {{expected '(' after 'initializer'}} -// expected-error@-7 {{expected variable name or 'this' in lambda capture list}} -// expected-error@-8 {{expected ')'}} -// expected-note@-9 {{to match this '('}} -#endif +// expected-error@-1 {{expected '(' after 'initializer'}} +// expected-error@-2 {{expected variable name or 'this' in lambda capture list}} +// expected-error@-3 {{expected ')'}} +// expected-note@-4 {{to match this '('}} #pragma omp declare reduction(fun4 : long : omp_out += omp_in) initializer() // expected-error {{expected expression}} #pragma omp declare reduction(fun5 : long : omp_out += omp_in) initializer(temp) // expected-error {{only 'omp_priv' or 'omp_orig' variables are allowed in initializer expression}} #pragma omp declare reduction(fun6 : long : omp_out += omp_in) initializer(omp_orig // expected-error {{expected ')'}} expected-note {{to match this '('}} diff --git a/clang/test/OpenMP/openmp_check.cpp b/clang/test/OpenMP/openmp_check.cpp index 6a8dd17fc836..b52ce0c06692 100644 --- a/clang/test/OpenMP/openmp_check.cpp +++ b/clang/test/OpenMP/openmp_check.cpp @@ -18,7 +18,7 @@ int nested(int a) { auto F = [&]() { #if __cplusplus <= 199711L // expected-warning@-2 {{'auto' type specifier is a C++11 extension}} - // expected-error@-3 {{expected expression}} + // expected-warning@-3 {{lambdas are a C++11 extension}} #endif #pragma omp parallel diff --git a/clang/test/Parser/cxx03-lambda-extension.cpp b/clang/test/Parser/cxx03-lambda-extension.cpp new file mode 100644 index 000000000000..82ae7da30530 --- /dev/null +++ b/clang/test/Parser/cxx03-lambda-extension.cpp @@ -0,0 +1,5 @@ +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++03 %s + +void func() { + []() {}; // expected-warning {{lambdas are a C++11 extension}} +} diff --git a/clang/test/Parser/cxx0x-lambda-expressions.cpp b/clang/test/Parser/cxx0x-lambda-expressions.cpp index 72b315a497c0..a786a964163e 100644 --- a/clang/test/Parser/cxx0x-lambda-expressions.cpp +++ b/clang/test/Parser/cxx0x-lambda-expressions.cpp @@ -1,10 +1,15 @@ -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 -Wno-c99-designator %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++20 -Wno-c99-designator %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++23 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx14ext,cxx17ext,cxx20ext,cxx23ext -std=c++03 -Wno-c99-designator %s -Wno-c++11-extensions +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx14ext,cxx17ext,cxx20ext,cxx23ext -std=c++11 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx17ext,cxx20ext,cxx23ext -std=c++14 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx20ext,cxx23ext -std=c++17 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx23ext -std=c++20 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected -std=c++23 -Wno-c99-designator %s enum E { e }; +#if __cplusplus >= 201103L constexpr int id(int n) { return n; } +#endif class C { @@ -19,28 +24,25 @@ class C { [&,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}} [=,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}} [] {}; - [=] (int i) {}; - [&] (int) mutable -> void {}; - [foo,bar] () { return 3; }; - [=,&foo] () {}; - [&,foo] () {}; - [this] () {}; + [=] (int i) {}; + [&] (int) mutable -> void {}; + [foo,bar] () { return 3; }; + [=,&foo] () {}; + [&,foo] () {}; + [this] () {}; [] () -> class C { return C(); }; [] () -> enum E { return e; }; - [] -> int { return 0; }; - [] mutable -> int { return 0; }; -#if __cplusplus <= 202002L - // expected-warning@-3 {{lambda without a parameter clause is a C++23 extension}} - // expected-warning@-3 {{is a C++23 extension}} -#endif + [] -> int { return 0; }; // cxx23ext-warning {{lambda without a parameter clause is a C++23 extension}} + [] mutable -> int { return 0; }; // cxx23ext-warning {{is a C++23 extension}} + [](int) -> {}; // PR13652 expected-error {{expected a type}} return 1; } void designator_or_lambda() { - typedef int T; - const int b = 0; + typedef int T; + const int b = 0; const int c = 1; int d; int a1[1] = {[b] (T()) {}}; // expected-error{{no viable conversion from '(lambda}} @@ -49,19 +51,18 @@ class C { int a4[1] = {[&b] = 1 }; // expected-error{{integral constant expression must have integral or unscoped enumeration type, not 'const int *'}} int a5[3] = { []{return 0;}() }; int a6[1] = {[this] = 1 }; // expected-error{{integral constant expression must have integral or unscoped enumeration type, not 'C *'}} - int a7[1] = {[d(0)] { return d; } ()}; - int a8[1] = {[d = 0] { return d; } ()}; - int a10[1] = {[id(0)] { return id; } ()}; -#if __cplusplus <= 201103L - // expected-warning@-4{{extension}} - // expected-warning@-4{{extension}} - // expected-warning@-4{{extension}} + int a7[1] = {[d(0)] { return d; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + int a8[1] = {[d = 0] { return d; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} +#if __cplusplus >= 201103L + int a10[1] = {[id(0)] { return id; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} #endif int a9[1] = {[d = 0] = 1}; // expected-error{{is not an integral constant expression}} #if __cplusplus >= 201402L // expected-note@-2{{constant expression cannot modify an object that is visible outside that expression}} #endif +#if __cplusplus >= 201103L int a11[1] = {[id(0)] = 1}; +#endif } void delete_lambda(int *p) { @@ -80,43 +81,33 @@ class C { // We support init-captures in C++11 as an extension. int z; void init_capture() { - [n(0)] () mutable -> int { return ++n; }; - [n{0}] { return; }; - [a([&b = z]{})](){}; - [n = 0] { return ++n; }; // expected-error {{captured by copy in a non-mutable}} - [n = {0}] { return; }; // expected-error {{}} -#if __cplusplus <= 201103L - // expected-warning@-6{{extension}} - // expected-warning@-6{{extension}} - // expected-warning@-6{{extension}} - // expected-warning@-7{{extension}} - // expected-warning@-7{{extension}} - // expected-warning@-7{{extension}} -#endif + [n(0)] () mutable -> int { return ++n; }; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + [n{0}] { return; }; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + [a([&b = z]{})](){}; // cxx14ext-warning 2 {{initialized lambda captures are a C++14 extension}} + [n = 0] { return ++n; }; // expected-error {{captured by copy in a non-mutable}} + // cxx14ext-warning@-1 {{initialized lambda captures are a C++14 extension}} + [n = {0}] { return; }; // expected-error {{}} + // cxx14ext-warning@-1 {{initialized lambda captures are a C++14 extension}} int x = 4; - auto y = [&r = x, x = x + 1]() -> int { -#if __cplusplus <= 201103L - // expected-warning@-2{{extension}} - // expected-warning@-3{{extension}} -#endif + auto y = [&r = x, x = x + 1]() -> int { // cxx14ext-warning 2 {{initialized lambda captures are a C++14 extension}} r += 2; return x + 2; } (); } void attributes() { - [] __attribute__((noreturn)){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{is a C++23 extension}} -#endif + [] __attribute__((noreturn)){}; // cxx23ext-warning {{lambda without a parameter clause is a C++23 extension}} + []() [[]] mutable {}; // expected-error {{expected body of lambda expression}} []() [[]] {}; []() [[]] -> void {}; []() mutable [[]] -> void {}; +#if __cplusplus >= 201103L []() mutable noexcept [[]] -> void {}; +#endif // Testing GNU-style attributes on lambdas -- the attribute is specified // before the mutable specifier instead of after (unlike C++11). @@ -126,28 +117,18 @@ class C { // Testing support for P2173 on adding attributes to the declaration // rather than the type. - [][[]](){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif -#if __cplusplus > 201703L - [][[]](){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif -#endif - [][[]]{}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif + [][[]](){}; // cxx23ext-warning {{an attribute specifier sequence in this position is a C++23 extension}} + + [][[]](){}; // cxx20ext-warning {{explicit template parameter list for lambdas is a C++20 extension}} + // cxx23ext-warning@-1 {{an attribute specifier sequence in this position is a C++23 extension}} + + [][[]]{}; // cxx23ext-warning {{an attribute specifier sequence in this position is a C++23 extension}} } void missing_parens() { - [] mutable {}; - [] noexcept {}; -#if __cplusplus <= 202002L - // expected-warning@-3 {{is a C++23 extension}} - // expected-warning@-3 {{is a C++23 extension}} + [] mutable {}; // cxx23ext-warning {{is a C++23 extension}} +#if __cplusplus >= 201103L + [] noexcept {}; // cxx23ext-warning {{is a C++23 extension}} #endif } }; @@ -165,10 +146,7 @@ struct A { }; struct S { - void mf() { A{[*this]{}}; } -#if __cplusplus < 201703L - // expected-warning@-2 {{C++17 extension}} -#endif + void mf() { A(([*this]{})); } // cxx17ext-warning {{'*this' by copy is a C++17 extension}} }; } diff --git a/clang/test/Parser/cxx2b-lambdas.cpp b/clang/test/Parser/cxx2b-lambdas.cpp index ad975a17b6e4..758ec9a42f56 100644 --- a/clang/test/Parser/cxx2b-lambdas.cpp +++ b/clang/test/Parser/cxx2b-lambdas.cpp @@ -1,30 +1,48 @@ +// RUN: %clang_cc1 -std=c++03 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions -Wno-c++11-extensions +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11 -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions +// RUN: %clang_cc1 -std=c++14 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions +// RUN: %clang_cc1 -std=c++17 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions +// RUN: %clang_cc1 -std=c++20 %s -verify -Wno-c++23-extensions // RUN: %clang_cc1 -std=c++23 %s -verify auto LL0 = [] {}; auto LL1 = []() {}; auto LL2 = []() mutable {}; -auto LL3 = []() constexpr {}; +#if __cplusplus >= 201103L +auto LL3 = []() constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif -auto L0 = [] constexpr {}; +#if __cplusplus >= 201103L +auto L0 = [] constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L1 = [] mutable {}; +#if __cplusplus >= 201103L auto L2 = [] noexcept {}; -auto L3 = [] constexpr mutable {}; -auto L4 = [] mutable constexpr {}; -auto L5 = [] constexpr mutable noexcept {}; +auto L3 = [] constexpr mutable {}; // cxx11-error {{return type 'void' is not a literal type}} +auto L4 = [] mutable constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +auto L5 = [] constexpr mutable noexcept {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L6 = [s = 1] mutable {}; -auto L7 = [s = 1] constexpr mutable noexcept {}; +#if __cplusplus >= 201103L +auto L7 = [s = 1] constexpr mutable noexcept {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L8 = [] -> bool { return true; }; auto L9 = [] { return true; }; +#if __cplusplus >= 201103L auto L10 = [] noexcept { return true; }; +#endif auto L11 = [] -> bool { return true; }; +#if __cplusplus >= 202002L auto L12 = [] consteval {}; auto L13 = []() requires true {}; // expected-error{{non-templated function cannot have a requires clause}} auto L14 = [] requires true() requires true {}; auto L15 = [] requires true noexcept {}; +#endif auto L16 = [] [[maybe_unused]]{}; -auto XL0 = [] mutable constexpr mutable {}; // expected-error{{cannot appear multiple times}} -auto XL1 = [] constexpr mutable constexpr {}; // expected-error{{cannot appear multiple times}} +#if __cplusplus >= 201103L +auto XL0 = [] mutable constexpr mutable {}; // expected-error{{cannot appear multiple times}} cxx11-error {{return type 'void' is not a literal type}} +auto XL1 = [] constexpr mutable constexpr {}; // expected-error{{cannot appear multiple times}} cxx11-error {{return type 'void' is not a literal type}} auto XL2 = []) constexpr mutable constexpr {}; // expected-error{{expected body of lambda expression}} auto XL3 = []( constexpr mutable constexpr {}; // expected-error{{invalid storage class specifier}} \ // expected-error{{function parameter cannot be constexpr}} \ @@ -33,16 +51,23 @@ auto XL3 = []( constexpr mutable constexpr {}; // expected-error{{invalid storag // expected-note{{to match this '('}} \ // expected-error{{expected body}} \ // expected-warning{{duplicate 'constexpr'}} +#endif // http://llvm.org/PR49736 auto XL4 = [] requires true {}; // expected-error{{expected body}} +#if __cplusplus >= 201703L auto XL5 = [] requires true requires true {}; // expected-error{{expected body}} auto XL6 = [] requires true noexcept requires true {}; // expected-error{{expected body}} +#endif auto XL7 = []() static static {}; // expected-error {{cannot appear multiple times}} auto XL8 = []() static mutable {}; // expected-error {{cannot be both mutable and static}} +#if __cplusplus >= 202002L auto XL9 = []() static consteval {}; -auto XL10 = []() static constexpr {}; +#endif +#if __cplusplus >= 201103L +auto XL10 = []() static constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto XL11 = [] static {}; auto XL12 = []() static {}; @@ -67,6 +92,7 @@ void static_captures() { }; } +#if __cplusplus >= 201703L constexpr auto static_capture_constexpr() { char n = 'n'; return [n] static { return n; }(); // expected-error {{a static lambda cannot have any captures}} @@ -78,3 +104,4 @@ constexpr auto capture_constexpr() { return [n] { return n; }(); } static_assert(capture_constexpr()); +#endif diff --git a/clang/test/Parser/objcxx-lambda-expressions-neg.mm b/clang/test/Parser/objcxx-lambda-expressions-neg.mm index b2fe39dfbf70..795157816dcf 100644 --- a/clang/test/Parser/objcxx-lambda-expressions-neg.mm +++ b/clang/test/Parser/objcxx-lambda-expressions-neg.mm @@ -1,13 +1,8 @@ // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++98 %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=cxx03 -std=c++98 %s // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 %s int main() { - []{}; -#if __cplusplus <= 199711L - // expected-error@-2 {{expected expression}} -#else + []{}; // cxx03-warning {{lambdas are a C++11 extension}} // expected-no-diagnostics -#endif - } diff --git a/clang/test/ParserHLSL/group_shared.hlsl b/clang/test/ParserHLSL/group_shared.hlsl index 0b9f28395ee4..44f3a2e5b450 100644 --- a/clang/test/ParserHLSL/group_shared.hlsl +++ b/clang/test/ParserHLSL/group_shared.hlsl @@ -3,8 +3,8 @@ extern groupshared float f; extern float groupshared f; // Ok, redeclaration? -// NOTE:lambda is not enabled except for hlsl202x. -// expected-error@+2 {{expected expression}} +// expected-warning@+3 {{lambdas are a C++11 extension}} +// expected-error@+2 {{expected body of lambda expression}} // expected-warning@+1 {{'auto' type specifier is a C++11 extension}} auto l = []() groupshared {}; diff --git a/clang/test/SemaCXX/cxx2a-template-lambdas.cpp b/clang/test/SemaCXX/cxx2a-template-lambdas.cpp index 7ac481369891..fff524e77d3b 100644 --- a/clang/test/SemaCXX/cxx2a-template-lambdas.cpp +++ b/clang/test/SemaCXX/cxx2a-template-lambdas.cpp @@ -1,10 +1,14 @@ -// RUN: %clang_cc1 -std=c++2a -verify %s +// RUN: %clang_cc1 -std=c++03 -verify -Dstatic_assert=_Static_assert -Wno-c++11-extensions -Wno-c++14-extensions -Wno-c++17-extensions -Wno-c++20-extensions %s +// RUN: %clang_cc1 -std=c++11 -verify=expected,cxx11,cxx11-cxx14 -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions %s +// RUN: %clang_cc1 -std=c++14 -verify=expected,cxx11-cxx14,cxx14 -Wno-c++20-extensions -Wno-c++17-extensions %s +// RUN: %clang_cc1 -std=c++17 -verify -Wno-c++20-extensions %s +// RUN: %clang_cc1 -std=c++20 -verify %s template -constexpr bool is_same = false; +inline const bool is_same = false; template -constexpr bool is_same = true; +inline const bool is_same = true; template struct DummyTemplate { }; @@ -23,7 +27,7 @@ void func() { L1.operator()<6>(); // expected-note {{in instantiation}} auto L2 = [] class T, class U>(T &&arg) { - static_assert(is_same, DummyTemplate>); // // expected-error {{static assertion failed}} + static_assert(is_same, DummyTemplate >); // // expected-error {{static assertion failed}} }; L2(DummyTemplate()); L2(DummyTemplate()); // expected-note {{in instantiation}} @@ -36,15 +40,20 @@ struct ShadowMe { } }; +#if __cplusplus >= 201102L template constexpr T outer() { - return []() { return x; }.template operator()<123>(); // expected-error {{no matching member function}} \ - expected-note {{candidate template ignored}} + // FIXME: The C++11 error seems wrong + return []() { return x; }.template operator()<123>(); // expected-error {{no matching member function}} \ + expected-note {{candidate template ignored}} \ + cxx11-note {{non-literal type '' cannot be used in a constant expression}} \ + cxx14-note {{non-literal type}} } -static_assert(outer() == 123); +static_assert(outer() == 123); // cxx11-cxx14-error {{not an integral constant expression}} cxx11-cxx14-note {{in call}} template int *outer(); // expected-note {{in instantiation}} +#endif - +#if __cplusplus >= 202002L namespace GH62611 { template struct C { @@ -87,3 +96,4 @@ void foo() { } } +#endif diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp index 389002ab0e34..151d74f21d64 100644 --- a/clang/test/SemaCXX/lambda-expressions.cpp +++ b/clang/test/SemaCXX/lambda-expressions.cpp @@ -1,6 +1,7 @@ -// RUN: %clang_cc1 -std=c++11 -Wno-unused-value -fsyntax-only -verify=expected,expected-cxx14,cxx11 -fblocks %s -// RUN: %clang_cc1 -std=c++14 -Wno-unused-value -fsyntax-only -verify -verify=expected-cxx14 -fblocks %s -// RUN: %clang_cc1 -std=c++17 -Wno-unused-value -verify -ast-dump -fblocks %s | FileCheck %s +// RUN: %clang_cc1 -std=c++11 -Wno-unused-value -fsyntax-only -verify=expected,not-cxx03,cxx03-cxx11,cxx11,expected-cxx14 -fblocks %s +// RUN: %clang_cc1 -std=c++03 -Wno-unused-value -fsyntax-only -verify=expected,cxx03,cxx03-cxx11,expected-cxx14 -fblocks %s -Ddecltype=__decltype -Dstatic_assert=_Static_assert -Wno-c++11-extensions +// RUN: %clang_cc1 -std=c++14 -Wno-unused-value -fsyntax-only -verify=expected,not-cxx03,expected-cxx14 -fblocks %s +// RUN: %clang_cc1 -std=c++17 -Wno-unused-value -verify=expected,not-cxx03 -ast-dump -fblocks %s | FileCheck %s namespace std { class type_info; }; @@ -93,14 +94,14 @@ namespace ImplicitCapture { [] { return ref_i; }; // expected-error {{variable 'ref_i' cannot be implicitly captured in a lambda with no capture-default specified}} expected-note {{lambda expression begins here}} expected-note 2 {{capture 'ref_i' by}} expected-note 2 {{default capture by}} static int j; - int &ref_j = j; - [] { return ref_j; }; // ok + int &ref_j = j; // cxx03-note {{declared here}} + [] { return ref_j; }; // cxx03-error {{variable 'ref_j' cannot be implicitly captured in a lambda with no capture-default specified}} cxx03-note 4 {{capture}} cxx03-note {{lambda expression begins here}} } } namespace SpecialMembers { void f() { - auto a = []{}; // expected-note 2{{here}} expected-note 2{{candidate}} + auto a = []{}; // expected-note 2{{here}} expected-note {{candidate}} not-cxx03-note {{candidate}} decltype(a) b; // expected-error {{no matching constructor}} decltype(a) c = a; decltype(a) d = static_cast(a); @@ -213,7 +214,7 @@ namespace VariadicPackExpansion { }; template void local_class() { - sink { + sink s( [] (Ts t) { struct S : Ts { void f(Ts t) { @@ -226,7 +227,7 @@ namespace VariadicPackExpansion { s.f(t); return s; } (Ts()).g() ... - }; + ); }; struct X {}; struct Y {}; template void local_class(); @@ -296,7 +297,7 @@ namespace PR16708 { namespace TypeDeduction { struct S {}; void f() { - const S s {}; + const S s = S(); S &&t = [&] { return s; } (); #if __cplusplus > 201103L S &&u = [&] () -> auto { return s; } (); @@ -308,7 +309,7 @@ namespace TypeDeduction { namespace lambdas_in_NSDMIs { template struct L { - T t{}; + T t = T(); T t2 = ([](int a) { return [](int b) { return b; };})(t)(t); }; L l; @@ -345,6 +346,7 @@ namespace CaptureIncomplete { } } +#if __cplusplus >= 201103L namespace CaptureAbstract { struct S { virtual void f() = 0; // expected-note {{unimplemented}} @@ -362,6 +364,7 @@ namespace CaptureAbstract { [=] { return s.n; }; // expected-error {{abstract}} } } +#endif namespace PR18128 { auto l = [=]{}; // expected-error {{non-local lambda expression cannot have a capture-default}} @@ -372,6 +375,8 @@ namespace PR18128 { // expected-error@-1 {{non-local lambda expression cannot have a capture-default}} // expected-error@-2 {{invalid use of non-static data member 'n'}} // expected-cxx14-error@-3 {{a lambda expression may not appear inside of a constant expression}} + // cxx03-error@-4 {{function declaration cannot have variably modified type}} + // cxx03-warning@-5 {{variable length arrays in C++ are a Clang extension}} int g(int k = ([=]{ return n; }(), 0)); // expected-error@-1 {{non-local lambda expression cannot have a capture-default}} // expected-error@-2 {{invalid use of non-static data member 'n'}} @@ -434,13 +439,13 @@ struct A { template void g(F f) { - auto a = A{}; + auto a = A(); // expected-note@-1 {{in instantiation of template class 'PR20731::A' requested here}} auto xf = [a, f]() {}; int x = sizeof(xf); }; void f() { - g([] {}); + g([] {}); // cxx03-warning {{template argument uses local type}} // expected-note-re@-1 {{in instantiation of function template specialization 'PR20731::g<(lambda at {{.*}}>' requested here}} } @@ -491,8 +496,8 @@ namespace PR21857 { fun() = default; using Fn::operator(); }; - template fun wrap(Fn fn); - auto x = wrap([](){}); + template fun wrap(Fn fn); // cxx03-warning {{template argument uses unnamed type}} + auto x = wrap([](){}); // cxx03-warning {{template argument uses unnamed type}} cxx03-note 2 {{unnamed type used in template argument was declared here}} } namespace PR13987 { @@ -559,8 +564,8 @@ struct B { int x; A a = [&] { int y = x; }; A b = [&] { [&] { [&] { int y = x; }; }; }; - A d = [&](auto param) { int y = x; }; // cxx11-error {{'auto' not allowed in lambda parameter}} - A e = [&](auto param) { [&] { [&](auto param2) { int y = x; }; }; }; // cxx11-error 2 {{'auto' not allowed in lambda parameter}} + A d = [&](auto param) { int y = x; }; // cxx03-cxx11-error {{'auto' not allowed in lambda parameter}} + A e = [&](auto param) { [&] { [&](auto param2) { int y = x; }; }; }; // cxx03-cxx11-error 2 {{'auto' not allowed in lambda parameter}} }; B b; @@ -588,9 +593,9 @@ struct S1 { }; void foo1() { - auto s0 = S1{[name=]() {}}; // expected-error 2 {{expected expression}} - auto s1 = S1{[name=name]() {}}; // expected-error {{use of undeclared identifier 'name'; did you mean 'name1'?}} - // cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} + auto s0 = S1([name=]() {}); // expected-error {{expected expression}} + auto s1 = S1([name=name]() {}); // expected-error {{use of undeclared identifier 'name'; did you mean 'name1'?}} + // cxx03-cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} } } @@ -606,7 +611,7 @@ namespace PR25627_dont_odr_use_local_consts { namespace ConversionOperatorDoesNotHaveDeducedReturnType { auto x = [](int){}; - auto y = [](auto &v) -> void { v.n = 0; }; // cxx11-error {{'auto' not allowed in lambda parameter}} cxx11-note {{candidate function not viable}} cxx11-note {{conversion candidate}} + auto y = [](auto &v) -> void { v.n = 0; }; // cxx03-cxx11-error {{'auto' not allowed in lambda parameter}} cxx03-cxx11-note {{candidate function not viable}} cxx03-cxx11-note {{conversion candidate}} using T = decltype(x); using U = decltype(y); using ExpectedTypeT = void (*)(int); @@ -626,14 +631,15 @@ namespace ConversionOperatorDoesNotHaveDeducedReturnType { template friend constexpr U::operator ExpectedTypeU() const noexcept; #else - friend auto T::operator()(int) const; // cxx11-error {{'auto' return without trailing return type; deduced return types are a C++14 extension}} + friend auto T::operator()(int) const; // cxx11-error {{'auto' return without trailing return type; deduced return types are a C++14 extension}} \ + cxx03-error {{'auto' not allowed in function return type}} friend T::operator ExpectedTypeT() const; template - friend void U::operator()(T&) const; // cxx11-error {{friend declaration of 'operator()' does not match any declaration}} + friend void U::operator()(T&) const; // cxx03-cxx11-error {{friend declaration of 'operator()' does not match any declaration}} // FIXME: This should not match, as above. template - friend U::operator ExpectedTypeU() const; // cxx11-error {{friend declaration of 'operator void (*)(type-parameter-0-0 &)' does not match any declaration}} + friend U::operator ExpectedTypeU() const; // cxx03-cxx11-error {{friend declaration of 'operator void (*)(type-parameter-0-0 &)' does not match any declaration}} #endif private: @@ -641,7 +647,7 @@ namespace ConversionOperatorDoesNotHaveDeducedReturnType { }; // Should be OK in C++14 and later: lambda's call operator is a friend. - void use(X &x) { y(x); } // cxx11-error {{no matching function for call to object}} + void use(X &x) { y(x); } // cxx03-cxx11-error {{no matching function for call to object}} // This used to crash in return type deduction for the conversion opreator. struct A { int n; void f() { +[](decltype(n)) {}; } }; @@ -682,8 +688,8 @@ namespace GH60518 { // function parameters that are used in enable_if struct StringLiteral { template -StringLiteral(const char (&array)[N]) - __attribute__((enable_if(__builtin_strlen(array) == 2, +StringLiteral(const char (&array)[N]) // cxx03-note {{declared here}} + __attribute__((enable_if(__builtin_strlen(array) == 2, // cxx03-error {{'enable_if' attribute expression never produces a constant expression}} cxx03-note {{read of variable}} "invalid string literal"))); }; @@ -695,7 +701,7 @@ StringLiteral(const char (&array)[N]) [[clang::annotate_type("test", array)]]; } void Func1() { - [[maybe_unused]] auto y = [&](decltype(StringLiteral("xx"))) {}; + [[maybe_unused]] auto y = [&](decltype(StringLiteral("xx"))) {}; // cxx03-note {{in instantiation of function template specialization}} [[maybe_unused]] auto z = [&](decltype(cpp_attribute::StringLiteral("xx"))) {}; } @@ -718,6 +724,7 @@ static_assert([]() constexpr { // Call operator attributes refering to a variable should // be properly handled after D124351 +#if __cplusplus >= 201103L constexpr int i = 2; void foo() { (void)[=][[gnu::aligned(i)]] () {}; // expected-warning{{C++23 extension}} @@ -725,15 +732,18 @@ void foo() { // CHECK-NEXT: ConstantExpr // CHECK-NEXT: value: Int 2 } +#endif void GH48527() { auto a = []()__attribute__((b(({ return 0; })))){}; // expected-warning {{unknown attribute 'b' ignored}} } +#if __cplusplus >= 201103L void GH67492() { constexpr auto test = 42; auto lambda = (test, []() noexcept(true) {}); } +#endif // FIXME: This currently causes clang to crash in C++11 mode. #if __cplusplus >= 201402L diff --git a/clang/test/SemaCXX/lambda-implicit-this-capture.cpp b/clang/test/SemaCXX/lambda-implicit-this-capture.cpp index 7e0e347a8fee..eb1f9e880aec 100644 --- a/clang/test/SemaCXX/lambda-implicit-this-capture.cpp +++ b/clang/test/SemaCXX/lambda-implicit-this-capture.cpp @@ -1,3 +1,4 @@ +// RUN: %clang_cc1 -std=c++03 -verify=cxx11 %s -Wno-c++11-extensions // RUN: %clang_cc1 -std=c++11 -verify=cxx11 %s // RUN: %clang_cc1 -std=c++2a -verify=cxx2a %s // RUN: %clang_cc1 -std=c++2a -verify=cxx2a-no-deprecated %s -Wno-deprecated diff --git a/clang/test/SemaCXX/lambda-invalid-capture.cpp b/clang/test/SemaCXX/lambda-invalid-capture.cpp index 236753871d70..5be8c8c5078f 100644 --- a/clang/test/SemaCXX/lambda-invalid-capture.cpp +++ b/clang/test/SemaCXX/lambda-invalid-capture.cpp @@ -1,3 +1,4 @@ +// RUN: %clang_cc1 -fsyntax-only -verify -std=c++03 -Wno-c++11-extensions %s // RUN: %clang_cc1 -fsyntax-only -verify %s // Don't crash. diff --git a/clang/test/SemaCXX/new-delete.cpp b/clang/test/SemaCXX/new-delete.cpp index 4f78b7c71a91..1a99c6aac604 100644 --- a/clang/test/SemaCXX/new-delete.cpp +++ b/clang/test/SemaCXX/new-delete.cpp @@ -171,12 +171,7 @@ void good_deletes() void bad_deletes() { delete 0; // expected-error {{cannot delete expression of type 'int'}} - delete [0] (int*)0; -#if __cplusplus <= 199711L - // expected-error@-2 {{expected expression}} -#else - // expected-error@-4 {{expected variable name or 'this' in lambda capture list}} -#endif + delete [0] (int*)0; // expected-error {{expected variable name or 'this' in lambda capture list}} delete (void*)0; // expected-warning {{cannot delete expression with pointer-to-'void' type 'void *'}} delete (T*)0; // expected-warning {{deleting pointer to incomplete type}} ::S::delete (int*)0; // expected-error {{expected unqualified-id}} -- GitLab From 4946cc37f4865b89fbebcfa0120183a11ae8d4ab Mon Sep 17 00:00:00 2001 From: Ilia Kuklin Date: Thu, 21 Mar 2024 17:05:35 +0500 Subject: [PATCH 036/933] [llvm-objcopy] Add --skip-symbol and --skip-symbols options (#80873) Add --skip-symbol and --skip-symbols options that allow to skip symbols when executing other options that can change the symbol's name, binding or visibility, similar to an existing option --keep-symbol that keeps a symbol from being removed by other options. --- llvm/docs/CommandGuide/llvm-objcopy.rst | 13 +++ llvm/docs/ReleaseNotes.rst | 4 + llvm/include/llvm/ObjCopy/CommonConfig.h | 1 + llvm/lib/ObjCopy/ConfigManager.cpp | 8 +- llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp | 3 + .../tools/llvm-objcopy/ELF/skip-symbol.test | 100 ++++++++++++++++++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp | 9 ++ llvm/tools/llvm-objcopy/ObjcopyOpts.td | 14 +++ 8 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst index 9d0cb7ad1195..985d16eb11cf 100644 --- a/llvm/docs/CommandGuide/llvm-objcopy.rst +++ b/llvm/docs/CommandGuide/llvm-objcopy.rst @@ -464,6 +464,19 @@ them. Read a list of symbols from and change their visibility to the specified value. Visibility values: default, internal, hidden, protected. +.. option:: --skip-symbol + + Do not change the parameters of symbol ```` when executing other + options that can change the symbol's name, binding or visibility. + +.. option:: --skip-symbols + + Do not change the parameters of symbols named in the file ```` when + executing other options that can change the symbol's name, binding or + visibility. In the file, each line represents a single symbol, with leading + and trailing whitespace ignored, as is anything following a '#'. + Can be specified multiple times to read names from multiple files. + .. option:: --split-dwo Equivalent to running :program:`llvm-objcopy` with :option:`--extract-dwo` and diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index 03691efe836f..01ecbdba5060 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -175,6 +175,10 @@ Changes to the LLVM tools ``--set-symbols-visibility`` options for ELF input to change the visibility of symbols. +* llvm-objcopy now supports ``--skip-symbol`` and ``--skip-symbols`` options + for ELF input to skip the specified symbols when executing other options + that can change a symbol's name, binding or visibility. + Changes to LLDB --------------------------------- diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h index 8f69c9fbeaf5..9d6d5fb23b18 100644 --- a/llvm/include/llvm/ObjCopy/CommonConfig.h +++ b/llvm/include/llvm/ObjCopy/CommonConfig.h @@ -233,6 +233,7 @@ struct CommonConfig { NameMatcher UnneededSymbolsToRemove; NameMatcher SymbolsToWeaken; NameMatcher SymbolsToKeepGlobal; + NameMatcher SymbolsToSkip; // Map options StringMap SectionsToRename; diff --git a/llvm/lib/ObjCopy/ConfigManager.cpp b/llvm/lib/ObjCopy/ConfigManager.cpp index e46b595a56dc..6442f1b958fb 100644 --- a/llvm/lib/ObjCopy/ConfigManager.cpp +++ b/llvm/lib/ObjCopy/ConfigManager.cpp @@ -15,7 +15,7 @@ namespace objcopy { Expected ConfigManager::getCOFFConfig() const { if (!Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || !Common.KeepSection.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToKeep.empty() || !Common.SymbolsToLocalize.empty() || !Common.SymbolsToWeaken.empty() || @@ -34,7 +34,7 @@ Expected ConfigManager::getCOFFConfig() const { Expected ConfigManager::getMachOConfig() const { if (!Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || !Common.KeepSection.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToKeep.empty() || !Common.SymbolsToLocalize.empty() || @@ -56,7 +56,7 @@ Expected ConfigManager::getMachOConfig() const { Expected ConfigManager::getWasmConfig() const { if (!Common.AddGnuDebugLink.empty() || Common.ExtractPartition || !Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || Common.DiscardMode != DiscardType::None || !Common.SymbolsToAdd.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToLocalize.empty() || @@ -77,7 +77,7 @@ Expected ConfigManager::getWasmConfig() const { Expected ConfigManager::getXCOFFConfig() const { if (!Common.AddGnuDebugLink.empty() || Common.ExtractPartition || !Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || Common.DiscardMode != DiscardType::None || !Common.AddSection.empty() || !Common.DumpSection.empty() || !Common.SymbolsToAdd.empty() || diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp index e4d6e02f3aa6..205bc1ef5b1a 100644 --- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp +++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp @@ -291,6 +291,9 @@ static Error updateAndRemoveSymbols(const CommonConfig &Config, return Error::success(); Obj.SymbolTable->updateSymbols([&](Symbol &Sym) { + if (Config.SymbolsToSkip.matches(Sym.Name)) + return; + // Common and undefined symbols don't make sense as local symbols, and can // even cause crashes if we localize those, so skip them. if (!Sym.isCommon() && Sym.getShndx() != SHN_UNDEF && diff --git a/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test new file mode 100644 index 000000000000..0f3ab808482b --- /dev/null +++ b/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test @@ -0,0 +1,100 @@ +## This test checks the functionality of options --skip-symbol and --skip-symbols. +# RUN: yaml2obj %s -o %t.o +# RUN: echo 'foo[2-3]' > %t.skip.regex + +## Check --skip-symbol functionality when changing symbol bindings. +# RUN: llvm-objcopy %t.o %t2.o --localize-hidden --skip-symbol=foo3 +# RUN: llvm-readelf -s %t2.o | FileCheck %s --check-prefix=LH-SYM +# LH-SYM-DAG: LOCAL HIDDEN 1 foo1 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo2 +# LH-SYM-DAG: GLOBAL HIDDEN 1 foo3 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo4 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo5 + +## Check --skip-symbols functionality when changing symbol bindings. +# RUN: llvm-objcopy %t.o %t1.o --localize-hidden --skip-symbols=%t.skip.regex --regex +# RUN: llvm-readelf -s %t1.o | FileCheck %s --check-prefix=LH-SYMS +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo1 +# LH-SYMS-DAG: GLOBAL HIDDEN 1 foo2 +# LH-SYMS-DAG: GLOBAL HIDDEN 1 foo3 +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo4 +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo5 + +## Check --skip-symbol functionality when changing symbol names. +# RUN: echo -e "foo1 bar1\nfoo2 bar2" > %t.renames.list +# RUN: llvm-objcopy %t.o %t4.o --redefine-syms=%t.renames.list \ +# RUN: --skip-symbol='fo*' --wildcard +# RUN: llvm-readelf -s %t4.o | FileCheck %s --check-prefix=RS-SYM +# RS-SYM-DAG: foo1 +# RS-SYM-DAG: foo2 +# RS-SYM-DAG: foo3 +# RS-SYM-DAG: foo4 +# RS-SYM-DAG: foo5 + +## Check --skip-symbols functionality when changing symbol names. +# RUN: llvm-objcopy %t.o %t3.o --redefine-syms=%t.renames.list \ +# RUN: --skip-symbols=%t.skip.regex --regex +# RUN: llvm-readelf -s %t3.o | FileCheck %s --check-prefix=RS-SYMS +# RS-SYMS-DAG: bar1 +# RS-SYMS-DAG: foo2 +# RS-SYMS-DAG: foo3 +# RS-SYMS-DAG: foo4 +# RS-SYMS-DAG: foo5 + +## Check the functionality when using skip options multiple times. +# RUN: echo "foo3" > %t.symbol0.list +# RUN: echo "foo4" > %t.symbol1.list +# RUN: llvm-objcopy %t.o %t5.o --set-symbol-visibility='foo*'=internal --wildcard \ +# RUN: --skip-symbol=foo1 --skip-symbol=foo2 \ +# RUN: --skip-symbols=%t.symbol0.list --skip-symbols=%t.symbol1.list +# RUN: llvm-readelf -s %t5.o | FileCheck %s --check-prefix=BOTH +# BOTH-DAG: GLOBAL HIDDEN 1 foo1 +# BOTH-DAG: GLOBAL HIDDEN 1 foo2 +# BOTH-DAG: GLOBAL HIDDEN 1 foo3 +# BOTH-DAG: GLOBAL HIDDEN 1 foo4 +## Only foo5 is not skipped. +# BOTH-DAG: GLOBAL INTERNAL 1 foo5 + +## Check that using an invalid symbol name regex generates an error. +# RUN: echo '*.' > %t.symbols.regex +# RUN: not llvm-objcopy %t.o --skip-symbols=%t.symbols.regex --regex 2>&1 | \ +# RUN: FileCheck %s --check-prefix=SYMBOL +# RUN: not llvm-objcopy %t.o --skip-symbol='*.' --regex 2>&1 | \ +# RUN: FileCheck %s --check-prefix=SYMBOL +# SYMBOL: error: cannot compile regular expression '*.': repetition-operator operand invalid + +## Check passing an invalid filename generates an error. +# RUN: not llvm-objcopy %t.o --skip-symbols=no_file 2>&1 | \ +# RUN: FileCheck %s --check-prefix=FILE -DMSG=%errc_ENOENT +# FILE: error: 'no_file': [[MSG]] + +!ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .text + Type: SHT_PROGBITS +Symbols: + - Name: foo1 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo2 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo3 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo4 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo5 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp index a0c6415bf0e6..7269c51a08d6 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp +++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp @@ -978,6 +978,15 @@ objcopy::parseObjcopyOptions(ArrayRef RawArgsArr, addSymbolsFromFile(Config.SymbolsToKeep, DC.Alloc, Arg->getValue(), SymbolMatchStyle, ErrorCallback)) return std::move(E); + for (auto *Arg : InputArgs.filtered(OBJCOPY_skip_symbol)) + if (Error E = Config.SymbolsToSkip.addMatcher(NameOrPattern::create( + Arg->getValue(), SymbolMatchStyle, ErrorCallback))) + return std::move(E); + for (auto *Arg : InputArgs.filtered(OBJCOPY_skip_symbols)) + if (Error E = + addSymbolsFromFile(Config.SymbolsToSkip, DC.Alloc, Arg->getValue(), + SymbolMatchStyle, ErrorCallback)) + return std::move(E); for (auto *Arg : InputArgs.filtered(OBJCOPY_add_symbol)) { Expected SymInfo = parseNewSymbolInfo(Arg->getValue()); if (!SymInfo) diff --git a/llvm/tools/llvm-objcopy/ObjcopyOpts.td b/llvm/tools/llvm-objcopy/ObjcopyOpts.td index 3c0e5cd475a3..be02616e8c68 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOpts.td +++ b/llvm/tools/llvm-objcopy/ObjcopyOpts.td @@ -206,6 +206,20 @@ defm keep_symbols "be repeated to read symbols from many files">, MetaVarName<"filename">; +defm skip_symbol : Eq<"skip-symbol", "Do not change parameters of symbol " + "when executing other options that can change the symbol's " + "name, binding or visibility">, + MetaVarName<"symbol">; + +defm skip_symbols + : Eq<"skip-symbols", + "Read a list of symbols from and run as if " + "--skip-symbol= is set for each one. " + "contains one symbol per line and may contain comments beginning with " + "'#'. Leading and trailing whitespace is stripped from each line. May " + "be repeated to read symbols from many files">, + MetaVarName<"filename">; + defm dump_section : Eq<"dump-section", "Dump contents of section named
into file ">, -- GitLab From 02cb89b36a7ae9be4ab657306b69dc9d2830d0d5 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Thu, 21 Mar 2024 20:09:49 +0800 Subject: [PATCH 037/933] [NewPM] Handle error in TargetPassRegistry.inc (#86112) Mistakenly believing that checking Expected is sufficient. --- llvm/include/llvm/Passes/TargetPassRegistry.inc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc index 50766a99f6a7..b618331c6998 100644 --- a/llvm/include/llvm/Passes/TargetPassRegistry.inc +++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc @@ -80,8 +80,10 @@ if (PopulateClassToPassNames) { #define ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (PassBuilder::checkParametrizedPassName(Name, NAME)) { \ auto Params = PassBuilder::parsePassParameters(PARSER, Name, NAME); \ - if (!Params) \ + if (!Params) { \ + errs() << NAME ": " << toString(Params.takeError()) << '\n'; \ return false; \ + } \ PM.addPass(CREATE_PASS(Params.get())); \ return true; \ } -- GitLab From 734026347cca85cf0e242ef5f04896f55e0ac113 Mon Sep 17 00:00:00 2001 From: Sergio Afonso Date: Thu, 21 Mar 2024 12:25:48 +0000 Subject: [PATCH 038/933] Reapply "[Flang][OpenMP][Lower] NFC: Move clause processing helpers into the ClauseProcessor (#85258)" (#85807) This patch contains slight modifications to the reverted PR #85258 to avoid issues with constructs containing multiple reduction clauses, uncovered by a test on the gfortran testsuite. This reverts commit 9f80444c2e669237a5c92013f1a42b91b5609012. --- flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 59 ++++++++++++-- flang/lib/Lower/OpenMP/ClauseProcessor.h | 15 ++-- flang/lib/Lower/OpenMP/OpenMP.cpp | 74 +++-------------- flang/lib/Lower/OpenMP/Utils.cpp | 19 +++++ flang/lib/Lower/OpenMP/Utils.h | 3 + .../Lower/OpenMP/wsloop-reduction-multi.f90 | 81 +++++++++++++++++++ 6 files changed, 171 insertions(+), 80 deletions(-) create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp index 95faa0767e36..52c3479b1ea9 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp @@ -208,6 +208,25 @@ addUseDeviceClause(Fortran::lower::AbstractConverter &converter, useDeviceSymbols.push_back(object.id()); } +static void convertLoopBounds(Fortran::lower::AbstractConverter &converter, + mlir::Location loc, + llvm::SmallVectorImpl &lowerBound, + llvm::SmallVectorImpl &upperBound, + llvm::SmallVectorImpl &step, + std::size_t loopVarTypeSize) { + fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); + // The types of lower bound, upper bound, and step are converted into the + // type of the loop variable if necessary. + mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize); + for (unsigned it = 0; it < (unsigned)lowerBound.size(); it++) { + lowerBound[it] = + firOpBuilder.createConvert(loc, loopVarType, lowerBound[it]); + upperBound[it] = + firOpBuilder.createConvert(loc, loopVarType, upperBound[it]); + step[it] = firOpBuilder.createConvert(loc, loopVarType, step[it]); + } +} + //===----------------------------------------------------------------------===// // ClauseProcessor unique clauses //===----------------------------------------------------------------------===// @@ -217,8 +236,7 @@ bool ClauseProcessor::processCollapse( llvm::SmallVectorImpl &lowerBound, llvm::SmallVectorImpl &upperBound, llvm::SmallVectorImpl &step, - llvm::SmallVectorImpl &iv, - std::size_t &loopVarTypeSize) const { + llvm::SmallVectorImpl &iv) const { bool found = false; fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); @@ -236,7 +254,7 @@ bool ClauseProcessor::processCollapse( found = true; } - loopVarTypeSize = 0; + std::size_t loopVarTypeSize = 0; do { Fortran::lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation(); @@ -267,6 +285,9 @@ bool ClauseProcessor::processCollapse( &*std::next(doConstructEval->getNestedEvaluations().begin()); } while (collapseValue > 0); + convertLoopBounds(converter, currentLocation, lowerBound, upperBound, step, + loopVarTypeSize); + return found; } @@ -902,17 +923,39 @@ bool ClauseProcessor::processMap( bool ClauseProcessor::processReduction( mlir::Location currentLocation, - llvm::SmallVectorImpl &reductionVars, - llvm::SmallVectorImpl &reductionDeclSymbols, - llvm::SmallVectorImpl *reductionSymbols) - const { + llvm::SmallVectorImpl &outReductionVars, + llvm::SmallVectorImpl &outReductionTypes, + llvm::SmallVectorImpl &outReductionDeclSymbols, + llvm::SmallVectorImpl + *outReductionSymbols) const { return findRepeatableClause( [&](const omp::clause::Reduction &clause, const Fortran::parser::CharBlock &) { + // Use local lists of reductions to prevent variables from other + // already-processed reduction clauses from impacting this reduction. + // For example, the whole `reductionVars` array is queried to decide + // whether to do the reduction byref. + llvm::SmallVector reductionVars; + llvm::SmallVector reductionDeclSymbols; + llvm::SmallVector reductionSymbols; ReductionProcessor rp; rp.addDeclareReduction(currentLocation, converter, clause, reductionVars, reductionDeclSymbols, - reductionSymbols); + outReductionSymbols ? &reductionSymbols + : nullptr); + + // Copy local lists into the output. + llvm::copy(reductionVars, std::back_inserter(outReductionVars)); + llvm::copy(reductionDeclSymbols, + std::back_inserter(outReductionDeclSymbols)); + if (outReductionSymbols) + llvm::copy(reductionSymbols, + std::back_inserter(*outReductionSymbols)); + + outReductionTypes.reserve(outReductionTypes.size() + + reductionVars.size()); + llvm::transform(reductionVars, std::back_inserter(outReductionTypes), + [](mlir::Value v) { return v.getType(); }); }); } diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h index ffa8a5e05593..1b76eb97e823 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.h +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h @@ -56,14 +56,12 @@ public: clauses(makeList(clauses, semaCtx)) {} // 'Unique' clauses: They can appear at most once in the clause list. - bool - processCollapse(mlir::Location currentLocation, - Fortran::lower::pft::Evaluation &eval, - llvm::SmallVectorImpl &lowerBound, - llvm::SmallVectorImpl &upperBound, - llvm::SmallVectorImpl &step, - llvm::SmallVectorImpl &iv, - std::size_t &loopVarTypeSize) const; + bool processCollapse( + mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval, + llvm::SmallVectorImpl &lowerBound, + llvm::SmallVectorImpl &upperBound, + llvm::SmallVectorImpl &step, + llvm::SmallVectorImpl &iv) const; bool processDefault() const; bool processDevice(Fortran::lower::StatementContext &stmtCtx, mlir::Value &result) const; @@ -126,6 +124,7 @@ public: bool processReduction(mlir::Location currentLocation, llvm::SmallVectorImpl &reductionVars, + llvm::SmallVectorImpl &reductionTypes, llvm::SmallVectorImpl &reductionDeclSymbols, llvm::SmallVectorImpl *reductionSymbols = nullptr) const; diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index d335129565b4..160ada379a08 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -214,24 +214,6 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter, firOpBuilder.restoreInsertionPoint(insPt); } -static mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, - std::size_t loopVarTypeSize) { - // OpenMP runtime requires 32-bit or 64-bit loop variables. - loopVarTypeSize = loopVarTypeSize * 8; - if (loopVarTypeSize < 32) { - loopVarTypeSize = 32; - } else if (loopVarTypeSize > 64) { - loopVarTypeSize = 64; - mlir::emitWarning(converter.getCurrentLocation(), - "OpenMP loop iteration variable cannot have more than 64 " - "bits size and will be narrowed into 64 bits."); - } - assert((loopVarTypeSize == 32 || loopVarTypeSize == 64) && - "OpenMP loop iteration variable size must be transformed into 32-bit " - "or 64-bit"); - return converter.getFirOpBuilder().getIntegerType(loopVarTypeSize); -} - static mlir::Operation * createAndSetPrivatizedLoopVar(Fortran::lower::AbstractConverter &converter, mlir::Location loc, mlir::Value indexVal, @@ -568,6 +550,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, mlir::omp::ClauseProcBindKindAttr procBindKindAttr; llvm::SmallVector allocateOperands, allocatorOperands, reductionVars; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; llvm::SmallVector reductionSymbols; @@ -578,13 +561,8 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, cp.processDefault(); cp.processAllocate(allocatorOperands, allocateOperands); if (!outerCombined) - cp.processReduction(currentLocation, reductionVars, reductionDeclSymbols, - &reductionSymbols); - - llvm::SmallVector reductionTypes; - reductionTypes.reserve(reductionVars.size()); - llvm::transform(reductionVars, std::back_inserter(reductionTypes), - [](mlir::Value v) { return v.getType(); }); + cp.processReduction(currentLocation, reductionVars, reductionTypes, + reductionDeclSymbols, &reductionSymbols); auto reductionCallback = [&](mlir::Operation *op) { llvm::SmallVector locs(reductionVars.size(), @@ -1465,25 +1443,6 @@ genOMP(Fortran::lower::AbstractConverter &converter, standaloneConstruct.u); } -static void convertLoopBounds(Fortran::lower::AbstractConverter &converter, - mlir::Location loc, - llvm::SmallVectorImpl &lowerBound, - llvm::SmallVectorImpl &upperBound, - llvm::SmallVectorImpl &step, - std::size_t loopVarTypeSize) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - // The types of lower bound, upper bound, and step are converted into the - // type of the loop variable if necessary. - mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize); - for (unsigned it = 0; it < (unsigned)lowerBound.size(); it++) { - lowerBound[it] = - firOpBuilder.createConvert(loc, loopVarType, lowerBound[it]); - upperBound[it] = - firOpBuilder.createConvert(loc, loopVarType, upperBound[it]); - step[it] = firOpBuilder.createConvert(loc, loopVarType, step[it]); - } -} - static llvm::SmallVector genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter, mlir::Location &loc, @@ -1517,7 +1476,7 @@ genLoopAndReductionVars( mlir::Location &loc, llvm::ArrayRef loopArgs, llvm::ArrayRef reductionArgs, - llvm::SmallVectorImpl &reductionTypes) { + llvm::ArrayRef reductionTypes) { fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); llvm::SmallVector blockArgTypes; @@ -1579,16 +1538,15 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, llvm::SmallVector lowerBound, upperBound, step, reductionVars; llvm::SmallVector alignedVars, nontemporalVars; llvm::SmallVector iv; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; mlir::omp::ClauseOrderKindAttr orderClauseOperand; mlir::IntegerAttr simdlenClauseOperand, safelenClauseOperand; - std::size_t loopVarTypeSize; ClauseProcessor cp(converter, semaCtx, loopOpClauseList); - cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv, - loopVarTypeSize); + cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv); cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); - cp.processReduction(loc, reductionVars, reductionDeclSymbols); + cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols); cp.processIf(clause::If::DirectiveNameModifier::Simd, ifClauseOperand); cp.processSimdlen(simdlenClauseOperand); cp.processSafelen(safelenClauseOperand); @@ -1598,9 +1556,6 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, Fortran::parser::OmpClause::Nontemporal, Fortran::parser::OmpClause::Order>(loc, ompDirective); - convertLoopBounds(converter, loc, lowerBound, upperBound, step, - loopVarTypeSize); - mlir::TypeRange resultType; auto simdLoopOp = firOpBuilder.create( loc, resultType, lowerBound, upperBound, step, alignedVars, @@ -1638,6 +1593,7 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, llvm::SmallVector lowerBound, upperBound, step, reductionVars; llvm::SmallVector linearVars, linearStepVars; llvm::SmallVector iv; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; llvm::SmallVector reductionSymbols; mlir::omp::ClauseOrderKindAttr orderClauseOperand; @@ -1645,20 +1601,15 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, mlir::UnitAttr nowaitClauseOperand, byrefOperand, scheduleSimdClauseOperand; mlir::IntegerAttr orderedClauseOperand; mlir::omp::ScheduleModifierAttr scheduleModClauseOperand; - std::size_t loopVarTypeSize; ClauseProcessor cp(converter, semaCtx, beginClauseList); - cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv, - loopVarTypeSize); + cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv); cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); - cp.processReduction(loc, reductionVars, reductionDeclSymbols, + cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols, &reductionSymbols); cp.processTODO(loc, ompDirective); - convertLoopBounds(converter, loc, lowerBound, upperBound, step, - loopVarTypeSize); - if (ReductionProcessor::doReductionByRef(reductionVars)) byrefOperand = firOpBuilder.getUnitAttr(); @@ -1699,11 +1650,6 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, auto *nestedEval = getCollapsedLoopEval( eval, Fortran::lower::getCollapseValue(beginClauseList)); - llvm::SmallVector reductionTypes; - reductionTypes.reserve(reductionVars.size()); - llvm::transform(reductionVars, std::back_inserter(reductionTypes), - [](mlir::Value v) { return v.getType(); }); - auto ivCallback = [&](mlir::Operation *op) { return genLoopAndReductionVars(op, converter, loc, iv, reductionSymbols, reductionTypes); diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp index fa4a51e33848..b9c0660aa4da 100644 --- a/flang/lib/Lower/OpenMP/Utils.cpp +++ b/flang/lib/Lower/OpenMP/Utils.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -70,6 +71,24 @@ void genObjectList2(const Fortran::parser::OmpObjectList &objectList, } } +mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, + std::size_t loopVarTypeSize) { + // OpenMP runtime requires 32-bit or 64-bit loop variables. + loopVarTypeSize = loopVarTypeSize * 8; + if (loopVarTypeSize < 32) { + loopVarTypeSize = 32; + } else if (loopVarTypeSize > 64) { + loopVarTypeSize = 64; + mlir::emitWarning(converter.getCurrentLocation(), + "OpenMP loop iteration variable cannot have more than 64 " + "bits size and will be narrowed into 64 bits."); + } + assert((loopVarTypeSize == 32 || loopVarTypeSize == 64) && + "OpenMP loop iteration variable size must be transformed into 32-bit " + "or 64-bit"); + return converter.getFirOpBuilder().getIntegerType(loopVarTypeSize); +} + void gatherFuncAndVarSyms( const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause, llvm::SmallVectorImpl &symbolAndClause) { diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h index 3ab0823a4621..4074bf73987d 100644 --- a/flang/lib/Lower/OpenMP/Utils.h +++ b/flang/lib/Lower/OpenMP/Utils.h @@ -51,6 +51,9 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc, mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy, bool isVal = false); +mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, + std::size_t loopVarTypeSize); + void gatherFuncAndVarSyms( const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause, llvm::SmallVectorImpl &symbolAndClause); diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 new file mode 100644 index 000000000000..9e9951c399c9 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 @@ -0,0 +1,81 @@ +! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[MIN_RED_I32_NAME:.*]] : i32 init { +!CHECK: ^bb0(%{{.*}}: i32): +!CHECK: %[[C0_1:.*]] = arith.constant 2147483647 : i32 +!CHECK: omp.yield(%[[C0_1]] : i32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32): +!CHECK: %[[RES:.*]] = arith.minsi %[[ARG0]], %[[ARG1]] : i32 +!CHECK: omp.yield(%[[RES]] : i32) +!CHECK: } + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[ADD_RED_F32_NAME:.*]] : f32 init { +!CHECK: ^bb0(%{{.*}}: f32): +!CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: omp.yield(%[[C0_1]] : f32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32): +!CHECK: %[[RES:.*]] = arith.addf %[[ARG0]], %[[ARG1]] {{.*}} : f32 +!CHECK: omp.yield(%[[RES]] : f32) +!CHECK: } + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[ADD_RED_I32_NAME:.*]] : i32 init { +!CHECK: ^bb0(%{{.*}}: i32): +!CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +!CHECK: omp.yield(%[[C0_1]] : i32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32): +!CHECK: %[[RES:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i32 +!CHECK: omp.yield(%[[RES]] : i32) +!CHECK: } + +!CHECK-LABEL: func.func @_QPmultiple_reduction +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_reductionEx"} +!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFmultiple_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Y_REF:.*]] = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFmultiple_reductionEy"} +!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFmultiple_reductionEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Z_REF:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_reductionEz"} +!CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z_REF]] {uniq_name = "_QFmultiple_reductionEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.wsloop reduction( +!CHECK-SAME: @[[ADD_RED_I32_NAME]] %[[X_DECL]]#0 -> %[[PRV_X:.+]] : !fir.ref, +!CHECK-SAME: @[[ADD_RED_F32_NAME]] %[[Y_DECL]]#0 -> %[[PRV_Y:.+]] : !fir.ref, +!CHECK-SAME: @[[MIN_RED_I32_NAME]] %[[Z_DECL]]#0 -> %[[PRV_Z:.+]] : !fir.ref) {{.*}}{ +!CHECK: %[[PRV_X_DECL:.+]]:2 = hlfir.declare %[[PRV_X]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRV_Y_DECL:.+]]:2 = hlfir.declare %[[PRV_Y]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRV_Z_DECL:.+]]:2 = hlfir.declare %[[PRV_Z]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[LPRV_X:.+]] = fir.load %[[PRV_X_DECL]]#0 : !fir.ref +!CHECK: %[[RES_X:.+]] = arith.addi %[[LPRV_X]], %{{.+}} : i32 +!CHECK: hlfir.assign %[[RES_X]] to %[[PRV_X_DECL]]#0 : i32, !fir.ref +!CHECK: %[[LPRV_Y:.+]] = fir.load %[[PRV_Y_DECL]]#0 : !fir.ref +!CHECK: %[[RES_Y:.+]] = arith.addf %[[LPRV_Y]], %{{.+}} : f32 +!CHECK: hlfir.assign %[[RES_Y]] to %[[PRV_Y_DECL]]#0 : f32, !fir.ref +!CHECK: %[[LPRV_Z:.+]] = fir.load %[[PRV_Z_DECL]]#0 : !fir.ref +!CHECK: %[[RES_Z:.+]] = arith.select %{{.+}}, %[[LPRV_Z]], %{{.+}} : i32 +!CHECK: hlfir.assign %[[RES_Z]] to %[[PRV_Z_DECL]]#0 : i32, !fir.ref +!CHECK: omp.yield +!CHECK: } +!CHECK: return +subroutine multiple_reduction(v) + implicit none + integer, intent(in) :: v(:) + integer :: i + integer :: x + real :: y + integer:: z + x = 0 + y = 0.0 + z = 10 + + !$omp do reduction(+:x,y) reduction(min:z) + do i=1, 100 + x = x + v(i) + y = y + 1.5 * v(i) + z = min(z, v(i)) + end do + !$omp end do +end subroutine -- GitLab From fa6e4338369c787710f1fe682cf6bd62348b9104 Mon Sep 17 00:00:00 2001 From: Spenser Bauman Date: Thu, 21 Mar 2024 09:02:21 -0400 Subject: [PATCH 039/933] [mlir][tosa] Fix assertion failure in tosa-layerwise-constant-fold (#85670) The existing implementation of tosa-layerwise-constant-fold only works for constant values backed by DenseElementsAttr. For constants which hold DenseResourceAttrs, the folder will end up asserting at runtime, as it assumes that the backing data can always be accessed through ElementsAttr::getValues. This change reworks the logic so that types types used to perform folding are based on whether the ElementsAttr can be converted to a range of that particular type. --------- Co-authored-by: Spenser Bauman Co-authored-by: Tina Jung --- .../Dialect/Tosa/Transforms/TosaFolders.cpp | 73 ++++++++++--------- mlir/test/Dialect/Tosa/constant-op-fold.mlir | 17 +++++ 2 files changed, 56 insertions(+), 34 deletions(-) diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp index 050f8ca3f32a..6575b39fd45a 100644 --- a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp +++ b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp @@ -132,14 +132,17 @@ bool constantUnaryOpShouldBeFolded(TosaOp unaryOp, DenseElementsAttr values) { return inputOp.hasOneUse(); } -template -DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, +template +DenseElementsAttr transposeType(const RangeType &data, ShapedType inputType, ShapedType outputType, llvm::ArrayRef permValues) { + using ElementType = std::decay_t; + + assert(inputType.getElementType() == outputType.getElementType()); + if (inputType.getNumElements() == 0) - return DenseElementsAttr::get(outputType, llvm::ArrayRef{}); + return DenseElementsAttr::get(outputType, llvm::ArrayRef{}); - auto attrValues = attr.getValues(); auto inputShape = inputType.getShape(); // The inverted permutation map and strides of the output are used to compute @@ -148,10 +151,11 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, auto outputStrides = computeStrides(outputType.getShape()); auto invertedPermValues = invertPermutationVector(permValues); - auto initialValue = *std::begin(attrValues); - SmallVector outputValues(inputType.getNumElements(), initialValue); + auto initialValue = *std::begin(data); + SmallVector outputValues(inputType.getNumElements(), + initialValue); - for (const auto &it : llvm::enumerate(attrValues)) { + for (const auto &it : llvm::enumerate(data)) { auto srcLinearIndex = it.index(); uint64_t dstLinearIndex = 0; @@ -170,7 +174,7 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, } return DenseElementsAttr::get(outputType, - llvm::ArrayRef(outputValues)); + llvm::ArrayRef(outputValues)); } // A type specialized transposition of an ElementsAttr. @@ -180,32 +184,28 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, DenseElementsAttr transpose(ElementsAttr attr, ShapedType inputType, ShapedType outputType, llvm::ArrayRef permValues) { - auto baseType = inputType.getElementType(); - - // Handle possible integer types - if (auto intType = dyn_cast(baseType)) { - switch (intType.getWidth()) { - case 1: - return transposeType(attr, inputType, outputType, permValues); - case 8: - return transposeType(attr, inputType, outputType, permValues); - case 16: - return transposeType(attr, inputType, outputType, permValues); - case 32: - return transposeType(attr, inputType, outputType, permValues); - case 64: - return transposeType(attr, inputType, outputType, permValues); - default: - return transposeType(attr, inputType, outputType, permValues); - } - } + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); - // Handle possible float types - if (baseType.isF32()) { - return transposeType(attr, inputType, outputType, permValues); - } + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); - return transposeType(attr, inputType, outputType, permValues); + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + return nullptr; } struct TosaFoldConstantTranspose : public OpRewritePattern { @@ -228,14 +228,19 @@ struct TosaFoldConstantTranspose : public OpRewritePattern { DenseIntElementsAttr permAttr; if (!matchPattern(op.getPerms(), m_Constant(&permAttr))) return failure(); - auto permValues = llvm::to_vector<6>(llvm::map_range( + auto permValues = llvm::map_to_vector( // TOSA allows both 32- and 64-bit integer tensors here. permAttr.getValues(), - [](const APInt &val) { return val.getSExtValue(); })); + [](const APInt &val) { return val.getSExtValue(); }); auto inputType = cast(op.getInput1().getType()); auto resultAttr = transpose(inputValues, inputType, outputType, permValues); + if (!resultAttr) { + return rewriter.notifyMatchFailure( + op, "unsupported attribute or element type"); + } + rewriter.replaceOpWithNewOp(op, outputType, resultAttr); return success(); } diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir index 27ca3ae3c21b..de752f31fcba 100644 --- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir +++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir @@ -112,6 +112,23 @@ func.func @transpose_nofold_quantized_types() -> tensor<1x1x2x2x!quant.uniform:f32:3, {1.000000e-01,1.000000e-01}>> } +// CHECK-LABEL: @transpose_nofold_dense_resource +func.func @transpose_nofold_dense_resource() -> tensor<2x2xf32> { + %0 = "tosa.const"() <{value = dense_resource : tensor<2x2xf32>}> : () -> tensor<2x2xf32> + %1 = "tosa.const"() <{value = dense<[1, 0]> : tensor<2xi32>}> : () -> tensor<2xi32> + + // CHECK: tosa.transpose + %2 = tosa.transpose %0, %1 : (tensor<2x2xf32>, tensor<2xi32>) -> tensor<2x2xf32> + return %2 : tensor<2x2xf32> +} +{-# + dialect_resources: { + builtin: { + resource: "0x08000000010000000000000002000000000000000300000000000000" + } + } +#-} + // ----- // CHECK-LABEL: @fold_add_zero_rhs_f32 -- GitLab From 49b520856967c2354339d3c2a05fcf1d2d637f30 Mon Sep 17 00:00:00 2001 From: Paul T Robinson Date: Thu, 21 Mar 2024 06:09:34 -0700 Subject: [PATCH 040/933] [X86][Headers] Specify result of NaN comparisons (#85862) Make sure all float/double comparison intrinsics specify what happens with a NaN input. Update some existing descriptions of comparison results to make them all consistent. Also replace "yields" with "returns" throughout. --- clang/lib/Headers/avxintrin.h | 42 +++++---- clang/lib/Headers/emmintrin.h | 166 ++++++++++++++++++++-------------- clang/lib/Headers/xmmintrin.h | 98 ++++++++++++++------ 3 files changed, 193 insertions(+), 113 deletions(-) diff --git a/clang/lib/Headers/avxintrin.h b/clang/lib/Headers/avxintrin.h index a8882e82e171..be7a0b247e03 100644 --- a/clang/lib/Headers/avxintrin.h +++ b/clang/lib/Headers/avxintrin.h @@ -207,6 +207,8 @@ _mm256_div_ps(__m256 __a, __m256 __b) /// Compares two 256-bit vectors of [4 x double] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPD instruction. @@ -226,6 +228,8 @@ _mm256_max_pd(__m256d __a, __m256d __b) /// Compares two 256-bit vectors of [8 x float] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPS instruction. @@ -245,6 +249,8 @@ _mm256_max_ps(__m256 __a, __m256 __b) /// Compares two 256-bit vectors of [4 x double] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPD instruction. @@ -264,6 +270,8 @@ _mm256_min_pd(__m256d __a, __m256d __b) /// Compares two 256-bit vectors of [8 x float] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPS instruction. @@ -1604,9 +1612,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Returns a [2 x double] vector consisting of two doubles corresponding to -/// the two comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1663,9 +1671,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// [4 x float], using the operation specified by the immediate integer /// operand. /// -/// Returns a [4 x float] vector consisting of four floats corresponding to -/// the four comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1721,9 +1729,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// 256-bit vectors of [4 x double], using the operation specified by the /// immediate integer operand. /// -/// Returns a [4 x double] vector consisting of four doubles corresponding to -/// the four comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1781,9 +1789,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// [8 x float], using the operation specified by the immediate integer /// operand. /// -/// Returns a [8 x float] vector consisting of eight floats corresponding to -/// the eight comparison results: zero if the comparison is false, and all -/// 1's if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1842,8 +1850,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// two 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// If the result is true, all 64 bits of the destination vector are set; -/// otherwise they are cleared. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1900,8 +1909,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// vectors of [4 x float], using the operation specified by the immediate /// integer operand. /// -/// If the result is true, all 32 bits of the destination vector are set; -/// otherwise they are cleared. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h index f0c2db752195..e85bfc47aa5c 100644 --- a/clang/lib/Headers/emmintrin.h +++ b/clang/lib/Headers/emmintrin.h @@ -259,6 +259,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_pd(__m128d __a) { /// result. The upper 64 bits of the result are copied from the upper /// double-precision value of the first operand. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINSD / MINSD instruction. @@ -278,9 +280,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_sd(__m128d __a, } /// Performs element-by-element comparison of the two 128-bit vectors of -/// [2 x double] and returns the vector containing the lesser of each pair of +/// [2 x double] and returns a vector containing the lesser of each pair of /// values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPD / MINPD instruction. @@ -301,6 +305,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_pd(__m128d __a, /// result. The upper 64 bits of the result are copied from the upper /// double-precision value of the first operand. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXSD / MAXSD instruction. @@ -320,9 +326,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_sd(__m128d __a, } /// Performs element-by-element comparison of the two 128-bit vectors of -/// [2 x double] and returns the vector containing the greater of each pair +/// [2 x double] and returns a vector containing the greater of each pair /// of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPD / MAXPD instruction. @@ -412,7 +420,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_xor_pd(__m128d __a, /// Compares each of the corresponding double-precision values of the /// 128-bit vectors of [2 x double] for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -432,7 +441,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpeq_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -452,7 +462,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmplt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -472,7 +483,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmple_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -492,7 +504,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpgt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -512,8 +525,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpge_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are ordered with respect to those in the second operand. /// -/// A pair of double-precision values are "ordered" with respect to each -/// other if neither value is a NaN. Each comparison yields 0x0 for false, +/// A pair of double-precision values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, /// 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile @@ -534,8 +547,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpord_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are unordered with respect to those in the second operand. /// -/// A pair of double-precision values are "unordered" with respect to each -/// other if one or both values are NaN. Each comparison yields 0x0 for +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for /// false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile @@ -557,7 +570,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpunord_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are unequal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -577,7 +591,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpneq_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -597,7 +612,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnlt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -617,7 +633,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnle_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -637,7 +654,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpngt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -656,7 +674,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnge_pd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -680,7 +699,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpeq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -704,7 +724,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmplt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -728,7 +749,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmple_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -753,7 +775,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpgt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -775,11 +798,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpge_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] to determine if -/// the value in the first parameter is "ordered" with respect to the +/// the value in the first parameter is ordered with respect to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair -/// of double-precision values are "ordered" with respect to each other if +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair +/// of double-precision values are ordered with respect to each other if /// neither value is a NaN. /// /// \headerfile @@ -801,11 +824,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpord_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] to determine if -/// the value in the first parameter is "unordered" with respect to the +/// the value in the first parameter is unordered with respect to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair -/// of double-precision values are "unordered" with respect to each other if +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair +/// of double-precision values are unordered with respect to each other if /// one or both values are NaN. /// /// \headerfile @@ -831,7 +854,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpunord_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -855,7 +879,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpneq_sd(__m128d __a, /// the value in the first parameter is not less than the corresponding /// value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -879,7 +904,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnlt_sd(__m128d __a, /// the value in the first parameter is not less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -903,7 +929,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnle_sd(__m128d __a, /// the value in the first parameter is not greater than the corresponding /// value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -928,7 +955,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpngt_sd(__m128d __a, /// the value in the first parameter is not greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -951,8 +979,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnge_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -975,8 +1003,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comieq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -999,8 +1027,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comilt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1023,8 +1051,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comile_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1047,8 +1075,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comigt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1071,8 +1099,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comige_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, 1 is returned. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -1093,8 +1121,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comineq_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1117,8 +1145,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomieq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1141,8 +1169,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomilt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1165,8 +1193,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomile_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1189,8 +1217,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomigt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1213,8 +1241,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomige_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, 1 is returned. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -3033,7 +3061,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi64(__m128i __a, /// Compares each of the corresponding 8-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3052,7 +3080,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi8(__m128i __a, /// Compares each of the corresponding 16-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3071,7 +3099,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi16(__m128i __a, /// Compares each of the corresponding 32-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -3091,7 +3119,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi32(__m128i __a, /// integer vectors to determine if the values in the first operand are /// greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3113,7 +3141,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi8(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3133,7 +3161,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi16(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -3153,7 +3181,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi32(__m128i __a, /// integer vectors to determine if the values in the first operand are less /// than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3173,7 +3201,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi8(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3193,7 +3221,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi16(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -4777,7 +4805,9 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) { /// 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -4811,7 +4841,9 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) { /// two 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h index b2c68c3b7be9..040194786a27 100644 --- a/clang/lib/Headers/xmmintrin.h +++ b/clang/lib/Headers/xmmintrin.h @@ -316,6 +316,8 @@ _mm_rsqrt_ps(__m128 __a) /// operands and returns the lesser value in the low-order bits of the /// vector of [4 x float]. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINSS / MINSS instructions. @@ -338,6 +340,8 @@ _mm_min_ss(__m128 __a, __m128 __b) /// Compares two 128-bit vectors of [4 x float] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPS / MINPS instructions. @@ -358,6 +362,8 @@ _mm_min_ps(__m128 __a, __m128 __b) /// operands and returns the greater value in the low-order bits of a 128-bit /// vector of [4 x float]. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXSS / MAXSS instructions. @@ -380,6 +386,8 @@ _mm_max_ss(__m128 __a, __m128 __b) /// Compares two 128-bit vectors of [4 x float] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPS / MAXPS instructions. @@ -478,6 +486,7 @@ _mm_xor_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -501,6 +510,7 @@ _mm_cmpeq_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] for equality. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -523,6 +533,7 @@ _mm_cmpeq_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -547,6 +558,7 @@ _mm_cmplt_ss(__m128 __a, __m128 __b) /// operand are less than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -569,6 +581,7 @@ _mm_cmplt_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in /// the low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -593,6 +606,7 @@ _mm_cmple_ss(__m128 __a, __m128 __b) /// operand are less than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -615,6 +629,7 @@ _mm_cmple_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -641,6 +656,7 @@ _mm_cmpgt_ss(__m128 __a, __m128 __b) /// operand are greater than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -663,6 +679,7 @@ _mm_cmpgt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -689,6 +706,7 @@ _mm_cmpge_ss(__m128 __a, __m128 __b) /// operand are greater than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -710,6 +728,7 @@ _mm_cmpge_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -734,6 +753,7 @@ _mm_cmpneq_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] for inequality. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -757,6 +777,7 @@ _mm_cmpneq_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -782,6 +803,7 @@ _mm_cmpnlt_ss(__m128 __a, __m128 __b) /// operand are not less than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -805,6 +827,7 @@ _mm_cmpnlt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -830,6 +853,7 @@ _mm_cmpnle_ss(__m128 __a, __m128 __b) /// operand are not less than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -853,6 +877,7 @@ _mm_cmpnle_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -880,6 +905,7 @@ _mm_cmpngt_ss(__m128 __a, __m128 __b) /// operand are not greater than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -903,6 +929,7 @@ _mm_cmpngt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -930,6 +957,7 @@ _mm_cmpnge_ss(__m128 __a, __m128 __b) /// operand are not greater than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -951,8 +979,9 @@ _mm_cmpnge_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is ordered with /// respect to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the -/// low-order bits of a vector of [4 x float]. +/// A pair of floating-point values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, +/// 0xFFFFFFFF for true. /// /// \headerfile /// @@ -977,7 +1006,9 @@ _mm_cmpord_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are ordered with respect to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// A pair of floating-point values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, +/// 0xFFFFFFFF for true. /// /// \headerfile /// @@ -999,8 +1030,9 @@ _mm_cmpord_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is unordered /// with respect to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the -/// low-order bits of a vector of [4 x float]. +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for +/// false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -1025,7 +1057,9 @@ _mm_cmpunord_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are unordered with respect to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for +/// false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// @@ -1046,8 +1080,8 @@ _mm_cmpunord_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both /// operands for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1071,8 +1105,8 @@ _mm_comieq_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is less than the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1096,8 +1130,8 @@ _mm_comilt_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is less than or equal to the /// second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1120,8 +1154,8 @@ _mm_comile_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is greater than the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1144,8 +1178,8 @@ _mm_comigt_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is greater than or equal to /// the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1168,8 +1202,8 @@ _mm_comige_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is not equal to the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -1191,8 +1225,8 @@ _mm_comineq_ss(__m128 __a, __m128 __b) /// Performs an unordered comparison of two 32-bit float values using /// the low-order bits of both operands to determine equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1215,8 +1249,8 @@ _mm_ucomieq_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// less than the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1239,8 +1273,8 @@ _mm_ucomilt_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// less than or equal to the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1263,8 +1297,8 @@ _mm_ucomile_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// greater than the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1287,8 +1321,8 @@ _mm_ucomigt_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// greater than or equal to the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1310,8 +1344,8 @@ _mm_ucomige_ss(__m128 __a, __m128 __b) /// Performs an unordered comparison of two 32-bit float values using /// the low-order bits of both operands to determine inequality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -3028,6 +3062,8 @@ _mm_movemask_ps(__m128 __a) /// operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -3061,6 +3097,8 @@ _mm_movemask_ps(__m128 __a) /// integer operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// -- GitLab From aa4cbaba1dd3882932ab8772392325242f3c7bee Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 06:08:40 -0700 Subject: [PATCH 041/933] [SLP][NFC]Add a test with @llvm.abs nodes, which can be analyzed for better bitwidth. --- .../X86/store-abs-minbitwidth.ll | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll new file mode 100644 index 000000000000..e8b854b7cea6 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx512vl -passes=slp-vectorizer -slp-threshold=-3 | FileCheck %s + + +define i32 @test(ptr noalias %in, ptr noalias %inn, ptr %out) { +; CHECK-LABEL: @test( +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[IN:%.*]], align 1 +; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_2]], align 1 +; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i8>, ptr [[INN:%.*]], align 1 +; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds i8, ptr [[INN]], i64 2 +; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i8>, ptr [[GEP_5]], align 1 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP3]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> [[TMP10]], <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i8> [[TMP11]] to <4 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = sub <4 x i32> [[TMP12]], [[TMP8]] +; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP13]], i1 true) +; CHECK-NEXT: [[TMP15:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i16> +; CHECK-NEXT: store <4 x i16> [[TMP15]], ptr [[OUT:%.*]], align 2 +; CHECK-NEXT: ret i32 undef +; + %load.1 = load i8, ptr %in, align 1 + %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 + %load.2 = load i8, ptr %gep.1, align 1 + %gep.2 = getelementptr inbounds i8, ptr %in, i64 2 + %load.3 = load i8, ptr %gep.2, align 1 + %gep.3 = getelementptr inbounds i8, ptr %in, i64 3 + %load.4 = load i8, ptr %gep.3, align 1 + %load.5 = load i8, ptr %inn, align 1 + %gep.4 = getelementptr inbounds i8, ptr %inn, i64 1 + %load.6 = load i8, ptr %gep.4, align 1 + %gep.5 = getelementptr inbounds i8, ptr %inn, i64 2 + %load.7 = load i8, ptr %gep.5, align 1 + %gep.6 = getelementptr inbounds i8, ptr %inn, i64 3 + %load.8 = load i8, ptr %gep.6, align 1 + %sext1 = sext i8 %load.1 to i32 + %sext2 = sext i8 %load.2 to i32 + %sext3 = sext i8 %load.3 to i32 + %sext4 = sext i8 %load.4 to i32 + %sext5 = sext i8 %load.5 to i32 + %sext6 = sext i8 %load.6 to i32 + %sext7 = sext i8 %load.7 to i32 + %sext8 = sext i8 %load.8 to i32 + %sub1 = sub i32 %sext1, %sext5 + %sub2 = sub i32 %sext2, %sext6 + %sub3 = sub i32 %sext7, %sext3 + %sub4 = sub i32 %sext8, %sext4 + %call1 = call i32 @llvm.abs(i32 %sub1, i1 true) + %call2 = call i32 @llvm.abs(i32 %sub2, i1 true) + %call3 = call i32 @llvm.abs(i32 %sub3, i1 true) + %call4 = call i32 @llvm.abs(i32 %sub4, i1 true) + %t1 = trunc i32 %call1 to i16 + %t2 = trunc i32 %call2 to i16 + %t3 = trunc i32 %call3 to i16 + %t4 = trunc i32 %call4 to i16 + %gep.8 = getelementptr inbounds i16, ptr %out, i64 1 + %gep.9 = getelementptr inbounds i16, ptr %out, i64 2 + %gep.10 = getelementptr inbounds i16, ptr %out, i64 3 + store i16 %t1, ptr %out, align 2 + store i16 %t2, ptr %gep.8, align 2 + store i16 %t3, ptr %gep.9, align 2 + store i16 %t4, ptr %gep.10, align 2 + + ret i32 undef +} -- GitLab From 538257bf00960f6134a51a17c8477b298ff87c30 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 14:24:55 +0100 Subject: [PATCH 042/933] [mlir][bazel] Update BUILD after 61b24c61a90802e06e40a7ab0aa5e2138486bd73 --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index e20aebe95063..201c7f653398 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3765,6 +3765,7 @@ cc_library( ":DialectUtils", ":IR", ":ShapedOpInterfaces", + ":SideEffectInterfaces", ":ViewLikeInterface", ":XeGPUIncGen", "//llvm:Core", -- GitLab From 0aa6d57e575dd920db81bef7ff509c4d3a9c6891 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 21 Mar 2024 14:27:37 +0100 Subject: [PATCH 043/933] [MLIR] Add initial convert-memref-to-emitc pass (#85389) This converts `memref.alloca`, `memref.load` & `memref.store` to `emitc.variable`, `emitc.subscript` and `emitc.assign`. --- .../Conversion/MemRefToEmitC/MemRefToEmitC.h | 21 ++++ .../MemRefToEmitC/MemRefToEmitCPass.h | 20 +++ mlir/include/mlir/Conversion/Passes.h | 1 + mlir/include/mlir/Conversion/Passes.td | 9 ++ mlir/lib/Conversion/CMakeLists.txt | 1 + .../Conversion/MemRefToEmitC/CMakeLists.txt | 18 +++ .../MemRefToEmitC/MemRefToEmitC.cpp | 114 ++++++++++++++++++ .../MemRefToEmitC/MemRefToEmitCPass.cpp | 55 +++++++++ .../MemRefToEmitC/memref-to-emitc-failed.mlir | 40 ++++++ .../MemRefToEmitC/memref-to-emitc.mlir | 28 +++++ .../llvm-project-overlay/mlir/BUILD.bazel | 27 +++++ 11 files changed, 334 insertions(+) create mode 100644 mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h create mode 100644 mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h create mode 100644 mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt create mode 100644 mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp create mode 100644 mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp create mode 100644 mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir create mode 100644 mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir diff --git a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h new file mode 100644 index 000000000000..734ffdba520c --- /dev/null +++ b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h @@ -0,0 +1,21 @@ +//===- MemRefToEmitC.h - Convert MemRef to EmitC --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H +#define MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H + +namespace mlir { +class RewritePatternSet; +class TypeConverter; + +void populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter); + +void populateMemRefToEmitCConversionPatterns(RewritePatternSet &patterns, + TypeConverter &converter); +} // namespace mlir + +#endif // MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H diff --git a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h new file mode 100644 index 000000000000..4a63014c19ad --- /dev/null +++ b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h @@ -0,0 +1,20 @@ +//===- MemRefToEmitCPass.h - A Pass to convert MemRef to EmitC ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H +#define MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H + +#include + +namespace mlir { +class Pass; + +#define GEN_PASS_DECL_CONVERTMEMREFTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +#endif // MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h index f2aa4fb53540..2179ae18ac07 100644 --- a/mlir/include/mlir/Conversion/Passes.h +++ b/mlir/include/mlir/Conversion/Passes.h @@ -45,6 +45,7 @@ #include "mlir/Conversion/MathToLLVM/MathToLLVM.h" #include "mlir/Conversion/MathToLibm/MathToLibm.h" #include "mlir/Conversion/MathToSPIRV/MathToSPIRVPass.h" +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h" #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h" #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRVPass.h" #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h" diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index bd81cc6d5323..7e7ee3a2f780 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -753,6 +753,15 @@ def ConvertMathToFuncs : Pass<"convert-math-to-funcs", "ModuleOp"> { ]; } +//===----------------------------------------------------------------------===// +// MemRefToEmitC +//===----------------------------------------------------------------------===// + +def ConvertMemRefToEmitC : Pass<"convert-memref-to-emitc"> { + let summary = "Convert MemRef dialect to EmitC dialect"; + let dependentDialects = ["emitc::EmitCDialect"]; +} + //===----------------------------------------------------------------------===// // MemRefToLLVM //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt index 8219cf98575f..41ab7046b91c 100644 --- a/mlir/lib/Conversion/CMakeLists.txt +++ b/mlir/lib/Conversion/CMakeLists.txt @@ -35,6 +35,7 @@ add_subdirectory(MathToFuncs) add_subdirectory(MathToLibm) add_subdirectory(MathToLLVM) add_subdirectory(MathToSPIRV) +add_subdirectory(MemRefToEmitC) add_subdirectory(MemRefToLLVM) add_subdirectory(MemRefToSPIRV) add_subdirectory(NVGPUToNVVM) diff --git a/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt b/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt new file mode 100644 index 000000000000..8a72e747d024 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt @@ -0,0 +1,18 @@ +add_mlir_conversion_library(MLIRMemRefToEmitC + MemRefToEmitC.cpp + MemRefToEmitCPass.cpp + + ADDITIONAL_HEADER_DIRS + ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/MemRefToEmitC + + DEPENDS + MLIRConversionPassIncGen + + LINK_COMPONENTS + Core + + LINK_LIBS PUBLIC + MLIREmitCDialect + MLIRMemRefDialect + MLIRTransforms + ) diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp new file mode 100644 index 000000000000..0e3b64692126 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp @@ -0,0 +1,114 @@ +//===- MemRefToEmitC.cpp - MemRef to EmitC conversion ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements patterns to convert memref ops into emitc ops. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h" + +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Transforms/DialectConversion.h" + +using namespace mlir; + +namespace { +struct ConvertAlloca final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::AllocaOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + if (!op.getType().hasStaticShape()) { + return rewriter.notifyMatchFailure( + op.getLoc(), "cannot transform alloca with dynamic shape"); + } + + if (op.getAlignment().value_or(1) > 1) { + // TODO: Allow alignment if it is not more than the natural alignment + // of the C array. + return rewriter.notifyMatchFailure( + op.getLoc(), "cannot transform alloca with alignment requirement"); + } + + auto resultTy = getTypeConverter()->convertType(op.getType()); + if (!resultTy) { + return rewriter.notifyMatchFailure(op.getLoc(), "cannot convert type"); + } + auto noInit = emitc::OpaqueAttr::get(getContext(), ""); + rewriter.replaceOpWithNewOp(op, resultTy, noInit); + return success(); + } +}; + +struct ConvertLoad final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::LoadOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + auto resultTy = getTypeConverter()->convertType(op.getType()); + if (!resultTy) { + return rewriter.notifyMatchFailure(op.getLoc(), "cannot convert type"); + } + + auto subscript = rewriter.create( + op.getLoc(), operands.getMemref(), operands.getIndices()); + + auto noInit = emitc::OpaqueAttr::get(getContext(), ""); + auto var = + rewriter.create(op.getLoc(), resultTy, noInit); + + rewriter.create(op.getLoc(), var, subscript); + rewriter.replaceOp(op, var); + return success(); + } +}; + +struct ConvertStore final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::StoreOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + auto subscript = rewriter.create( + op.getLoc(), operands.getMemref(), operands.getIndices()); + rewriter.replaceOpWithNewOp(op, subscript, + operands.getValue()); + return success(); + } +}; +} // namespace + +void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) { + typeConverter.addConversion( + [&](MemRefType memRefType) -> std::optional { + if (!memRefType.hasStaticShape() || + !memRefType.getLayout().isIdentity() || memRefType.getRank() == 0) { + return {}; + } + Type convertedElementType = + typeConverter.convertType(memRefType.getElementType()); + if (!convertedElementType) + return {}; + return emitc::ArrayType::get(memRefType.getShape(), + convertedElementType); + }); +} + +void mlir::populateMemRefToEmitCConversionPatterns(RewritePatternSet &patterns, + TypeConverter &converter) { + patterns.add(converter, + patterns.getContext()); +} diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp new file mode 100644 index 000000000000..4e5d1912d157 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp @@ -0,0 +1,55 @@ +//===- MemRefToEmitC.cpp - MemRef to EmitC conversion ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a pass to convert memref ops into emitc ops. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h" + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h" +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" + +namespace mlir { +#define GEN_PASS_DEF_CONVERTMEMREFTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +using namespace mlir; + +namespace { +struct ConvertMemRefToEmitCPass + : public impl::ConvertMemRefToEmitCBase { + void runOnOperation() override { + TypeConverter converter; + + // Fallback for other types. + converter.addConversion([](Type type) -> std::optional { + if (isa(type)) + return {}; + return type; + }); + + populateMemRefToEmitCTypeConversion(converter); + + RewritePatternSet patterns(&getContext()); + populateMemRefToEmitCConversionPatterns(patterns, converter); + + ConversionTarget target(getContext()); + target.addIllegalDialect(); + target.addLegalDialect(); + + if (failed(applyPartialConversion(getOperation(), target, + std::move(patterns)))) + return signalPassFailure(); + } +}; +} // namespace diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir new file mode 100644 index 000000000000..390190d341e5 --- /dev/null +++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir @@ -0,0 +1,40 @@ +// RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file -verify-diagnostics + +func.func @memref_op(%arg0 : memref<2x4xf32>) { + // expected-error@+1 {{failed to legalize operation 'memref.copy'}} + memref.copy %arg0, %arg0 : memref<2x4xf32> to memref<2x4xf32> + return +} + +// ----- + +func.func @alloca_with_dynamic_shape() { + %0 = index.constant 1 + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %1 = memref.alloca(%0) : memref<4x?xf32> + return +} + +// ----- + +func.func @alloca_with_alignment() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() {alignment = 64 : i64}: memref<4xf32> + return +} + +// ----- + +func.func @non_identity_layout() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() : memref<4x3xf32, affine_map<(d0, d1) -> (d1, d0)>> + return +} + +// ----- + +func.func @zero_rank() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() : memref + return +} diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir new file mode 100644 index 000000000000..9793b2d6d783 --- /dev/null +++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir @@ -0,0 +1,28 @@ +// RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file | FileCheck %s + +// CHECK-LABEL: memref_store +// CHECK-SAME: %[[v:.*]]: f32, %[[i:.*]]: index, %[[j:.*]]: index +func.func @memref_store(%v : f32, %i: index, %j: index) { + // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> + %0 = memref.alloca() : memref<4x8xf32> + + // CHECK: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: emitc.assign %[[v]] : f32 to %[[SUBSCRIPT:.*]] : f32 + memref.store %v, %0[%i, %j] : memref<4x8xf32> + return +} +// ----- + +// CHECK-LABEL: memref_load +// CHECK-SAME: %[[i:.*]]: index, %[[j:.*]]: index +func.func @memref_load(%i: index, %j: index) -> f32 { + // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> + %0 = memref.alloca() : memref<4x8xf32> + + // CHECK: %[[LOAD:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: %[[VAR:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32 + // CHECK: emitc.assign %[[LOAD]] : f32 to %[[VAR]] : f32 + %1 = memref.load %0[%i, %j] : memref<4x8xf32> + // CHECK: return %[[VAR]] : f32 + return %1 : f32 +} diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 201c7f653398..3b575d4a413c 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -4186,6 +4186,7 @@ cc_library( ":MathToLLVM", ":MathToLibm", ":MathToSPIRV", + ":MemRefToEmitC", ":MemRefToLLVM", ":MemRefToSPIRV", ":NVGPUToNVVM", @@ -8256,6 +8257,32 @@ cc_library( ], ) +cc_library( + name = "MemRefToEmitC", + srcs = glob([ + "lib/Conversion/MemRefToEmitC/*.cpp", + "lib/Conversion/MemRefToEmitC/*.h", + ]), + hdrs = glob([ + "include/mlir/Conversion/MemRefToEmitC/*.h", + ]), + includes = [ + "include", + "lib/Conversion/MemRefToEmitC", + ], + deps = [ + ":ConversionPassIncGen", + ":EmitCDialect", + ":MemRefDialect", + ":IR", + ":Pass", + ":Support", + ":TransformUtils", + ":Transforms", + "//llvm:Support", + ], +) + cc_library( name = "MemRefToLLVM", srcs = glob(["lib/Conversion/MemRefToLLVM/*.cpp"]), -- GitLab From 276283d8641f13b6ecde736ab1a8720f742d9d02 Mon Sep 17 00:00:00 2001 From: chrulski-intel Date: Thu, 21 Mar 2024 06:34:01 -0700 Subject: [PATCH 044/933] [LLD] [MinGW] Implement the -lto-sample-profile option (#85841) This has been a supported option for ELF and is added to the COFF Linker in #85701 --- lld/MinGW/Driver.cpp | 2 ++ lld/MinGW/Options.td | 2 ++ lld/test/MinGW/driver.test | 3 +++ 3 files changed, 7 insertions(+) diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp index efd643f9a322..bb08c77b2e11 100644 --- a/lld/MinGW/Driver.cpp +++ b/lld/MinGW/Driver.cpp @@ -455,6 +455,8 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, add("-lldemit:llvm"); if (args.hasArg(OPT_lto_emit_asm)) add("-lldemit:asm"); + if (auto *arg = args.getLastArg(OPT_lto_sample_profile)) + add("-lto-sample-profile:" + StringRef(arg->getValue())); if (auto *a = args.getLastArg(OPT_thinlto_cache_dir)) add("-lldltocache:" + StringRef(a->getValue())); diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td index 9a0a96aac7f1..56f67e3dd96c 100644 --- a/lld/MinGW/Options.td +++ b/lld/MinGW/Options.td @@ -160,6 +160,8 @@ def lto_cs_profile_file: JJ<"lto-cs-profile-file=">, HelpText<"Context sensitive profile file path">; def lto_emit_asm: FF<"lto-emit-asm">, HelpText<"Emit assembly code">; +def lto_sample_profile: JJ<"lto-sample-profile=">, + HelpText<"Sample profile file path">; def thinlto_cache_dir: JJ<"thinlto-cache-dir=">, HelpText<"Path to ThinLTO cached object file directory">; diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test index a4e9e5e1b19b..619fee8dee7c 100644 --- a/lld/test/MinGW/driver.test +++ b/lld/test/MinGW/driver.test @@ -422,6 +422,9 @@ LTO_EMIT_ASM: -lldemit:asm RUN: ld.lld -### foo.o -m i386pe -plugin-opt=emit-llvm 2>&1 | FileCheck -check-prefix=LTO_EMIT_LLVM %s LTO_EMIT_LLVM: -lldemit:llvm +RUN: ld.lld -### foo.o -m i386pep --lto-sample-profile=foo 2>&1 | FileCheck -check-prefix=LTO_SAMPLE_PROFILE %s +LTO_SAMPLE_PROFILE: -lto-sample-profile:foo + Test GCC specific LTO options that GCC passes unconditionally, that we ignore. RUN: ld.lld -### foo.o -m i386pep -plugin /usr/lib/gcc/x86_64-w64-mingw32/10-posix/liblto_plugin.so -plugin-opt=/usr/lib/gcc/x86_64-w64-mingw32/10-posix/lto-wrapper -plugin-opt=-fresolution=/tmp/ccM9d4fP.res -plugin-opt=-pass-through=-lmingw32 2> /dev/null -- GitLab From 5344a370fe85d9119729d9036540bbd91956da38 Mon Sep 17 00:00:00 2001 From: Kirill Chibisov Date: Thu, 21 Mar 2024 17:35:00 +0400 Subject: [PATCH 045/933] [mlir][emitc] Fix form-expressions inside expression (#86081) Make form-expressions not create `emitc.expression`s for operations inside the `emitc.expression`s, since they are invalid. --- .../EmitC/Transforms/FormExpressions.cpp | 3 ++- mlir/test/Dialect/EmitC/transforms.mlir | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp index 5b03f81b305f..e7c431f39e3f 100644 --- a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp +++ b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp @@ -36,7 +36,8 @@ struct FormExpressionsPass // Wrap each C operator op with an expression op. OpBuilder builder(context); auto matchFun = [&](Operation *op) { - if (op->hasTrait()) + if (op->hasTrait() && + !op->getParentOfType()) createExpression(op, builder); }; rootOp->walk(matchFun); diff --git a/mlir/test/Dialect/EmitC/transforms.mlir b/mlir/test/Dialect/EmitC/transforms.mlir index ad167fa455a1..8ac606a2c8c0 100644 --- a/mlir/test/Dialect/EmitC/transforms.mlir +++ b/mlir/test/Dialect/EmitC/transforms.mlir @@ -107,3 +107,20 @@ func.func @expression_with_address_taken(%arg0: i32, %arg1: i32, %arg2: !emitc.p %d = emitc.cmp lt, %c, %arg2 :(!emitc.ptr, !emitc.ptr) -> i1 return %d : i1 } + +// CHECK-LABEL: func.func @no_nested_expression( +// CHECK-SAME: %[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32) -> i1 { +// CHECK: %[[VAL_2:.*]] = emitc.expression : i1 { +// CHECK: %[[VAL_3:.*]] = emitc.cmp lt, %[[VAL_0]], %[[VAL_1]] : (i32, i32) -> i1 +// CHECK: emitc.yield %[[VAL_3]] : i1 +// CHECK: } +// CHECK: return %[[VAL_2]] : i1 +// CHECK: } + +func.func @no_nested_expression(%arg0: i32, %arg1: i32) -> i1 { + %a = emitc.expression : i1 { + %b = emitc.cmp lt, %arg0, %arg1 :(i32, i32) -> i1 + emitc.yield %b : i1 + } + return %a : i1 +} -- GitLab From 15eba9c12a1486ee600e35ecb83b1f2c8459416e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 12:48:19 +0000 Subject: [PATCH 046/933] [VectorCombine] Add DataLayout to VectorCombine class instead of repeated calls to getDataLayout(). NFC. --- .../Transforms/Vectorize/VectorCombine.cpp | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 23494314f132..7e86137f23f3 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -66,8 +66,8 @@ class VectorCombine { public: VectorCombine(Function &F, const TargetTransformInfo &TTI, const DominatorTree &DT, AAResults &AA, AssumptionCache &AC, - bool TryEarlyFoldsOnly) - : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), + const DataLayout *DL, bool TryEarlyFoldsOnly) + : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), DL(DL), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {} bool run(); @@ -79,6 +79,7 @@ private: const DominatorTree &DT; AAResults &AA; AssumptionCache &AC; + const DataLayout *DL; /// If true, only perform beneficial early IR transforms. Do not introduce new /// vector operations. @@ -181,7 +182,6 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { // We use minimal alignment (maximum flexibility) because we only care about // the dereferenceable region. When calculating cost and creating a new op, // we may use a larger value based on alignment attributes. - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts(); assert(isa(SrcPtr->getType()) && "Expected a pointer type"); @@ -189,15 +189,15 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { auto *MinVecTy = VectorType::get(ScalarTy, MinVecNumElts, false); unsigned OffsetEltIndex = 0; Align Alignment = Load->getAlign(); - if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), DL, Load, &AC, + if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), *DL, Load, &AC, &DT)) { // It is not safe to load directly from the pointer, but we can still peek // through gep offsets and check if it safe to load from a base address with // updated alignment. If it is, we can shuffle the element(s) into place // after loading. - unsigned OffsetBitWidth = DL.getIndexTypeSizeInBits(SrcPtr->getType()); + unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(SrcPtr->getType()); APInt Offset(OffsetBitWidth, 0); - SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL, Offset); + SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); // We want to shuffle the result down from a high element of a vector, so // the offset must be positive. @@ -215,7 +215,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { if (OffsetEltIndex >= MinVecNumElts) return false; - if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), DL, Load, &AC, + if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), *DL, Load, &AC, &DT)) return false; @@ -227,7 +227,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { // Original pattern: insertelt undef, load [free casts of] PtrOp, 0 // Use the greater of the alignment on the load or its source pointer. - Alignment = std::max(SrcPtr->getPointerAlignment(DL), Alignment); + Alignment = std::max(SrcPtr->getPointerAlignment(*DL), Alignment); Type *LoadTy = Load->getType(); unsigned AS = Load->getPointerAddressSpace(); InstructionCost OldCost = @@ -298,14 +298,13 @@ bool VectorCombine::widenSubvectorLoad(Instruction &I) { // the dereferenceable region. When calculating cost and creating a new op, // we may use a larger value based on alignment attributes. auto *Ty = cast(I.getType()); - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts(); assert(isa(SrcPtr->getType()) && "Expected a pointer type"); Align Alignment = Load->getAlign(); - if (!isSafeToLoadUnconditionally(SrcPtr, Ty, Align(1), DL, Load, &AC, &DT)) + if (!isSafeToLoadUnconditionally(SrcPtr, Ty, Align(1), *DL, Load, &AC, &DT)) return false; - Alignment = std::max(SrcPtr->getPointerAlignment(DL), Alignment); + Alignment = std::max(SrcPtr->getPointerAlignment(*DL), Alignment); Type *LoadTy = Load->getType(); unsigned AS = Load->getPointerAddressSpace(); @@ -854,7 +853,6 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) { // Scalarize the intrinsic ElementCount EC = cast(Op0->getType())->getElementCount(); Value *EVL = VPI.getArgOperand(3); - const DataLayout &DL = VPI.getModule()->getDataLayout(); // If the VP op might introduce UB or poison, we can scalarize it provided // that we know the EVL > 0: If the EVL is zero, then the original VP op @@ -867,7 +865,7 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) { else SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode( *FunctionalOpcode, &VPI, nullptr, &AC, &DT); - if (!SafeToSpeculate && !isKnownNonZero(EVL, DL, 0, &AC, &VPI, &DT)) + if (!SafeToSpeculate && !isKnownNonZero(EVL, *DL, 0, &AC, &VPI, &DT)) return false; Value *ScalarVal = @@ -1246,12 +1244,11 @@ bool VectorCombine::foldSingleElementStore(Instruction &I) { if (auto *Load = dyn_cast(Source)) { auto VecTy = cast(SI->getValueOperand()->getType()); - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts(); // Don't optimize for atomic/volatile load or store. Ensure memory is not // modified between, vector type matches store size, and index is inbounds. if (!Load->isSimple() || Load->getParent() != SI->getParent() || - !DL.typeSizeEqualsStoreSize(Load->getType()->getScalarType()) || + !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) || SrcAddr != SI->getPointerOperand()->stripPointerCasts()) return false; @@ -1270,7 +1267,7 @@ bool VectorCombine::foldSingleElementStore(Instruction &I) { NSI->copyMetadata(*SI); Align ScalarOpAlignment = computeAlignmentAfterScalarization( std::max(SI->getAlign(), Load->getAlign()), NewElement->getType(), Idx, - DL); + *DL); NSI->setAlignment(ScalarOpAlignment); replaceValue(I, *NSI); eraseInstruction(I); @@ -1288,8 +1285,7 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) { auto *VecTy = cast(I.getType()); auto *LI = cast(&I); - const DataLayout &DL = I.getModule()->getDataLayout(); - if (LI->isVolatile() || !DL.typeSizeEqualsStoreSize(VecTy->getScalarType())) + if (LI->isVolatile() || !DL->typeSizeEqualsStoreSize(VecTy->getScalarType())) return false; InstructionCost OriginalCost = @@ -1367,7 +1363,7 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) { VecTy->getElementType(), GEP, EI->getName() + ".scalar")); Align ScalarOpAlignment = computeAlignmentAfterScalarization( - LI->getAlign(), VecTy->getElementType(), Idx, DL); + LI->getAlign(), VecTy->getElementType(), Idx, *DL); NewLoad->setAlignment(ScalarOpAlignment); replaceValue(*EI, *NewLoad); @@ -2042,7 +2038,8 @@ PreservedAnalyses VectorCombinePass::run(Function &F, TargetTransformInfo &TTI = FAM.getResult(F); DominatorTree &DT = FAM.getResult(F); AAResults &AA = FAM.getResult(F); - VectorCombine Combiner(F, TTI, DT, AA, AC, TryEarlyFoldsOnly); + const DataLayout *DL = &F.getParent()->getDataLayout(); + VectorCombine Combiner(F, TTI, DT, AA, AC, DL, TryEarlyFoldsOnly); if (!Combiner.run()) return PreservedAnalyses::all(); PreservedAnalyses PA; -- GitLab From 686f4599cfa444aa62db4e22bf752f3d9614c30d Mon Sep 17 00:00:00 2001 From: David Green Date: Thu, 21 Mar 2024 13:45:44 +0000 Subject: [PATCH 047/933] [ARM] Regenerate some check lines. NFC --- llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll | 15 +- llvm/test/CodeGen/ARM/select.ll | 399 ++++++++++++++++-- 2 files changed, 361 insertions(+), 53 deletions(-) diff --git a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll index 365727c9dd27..0795525fba1b 100644 --- a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll +++ b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll @@ -8,10 +8,8 @@ %struct.Foo = type { ptr } -; ARM-LABEL: foo: -; THUMB-LABEL: foo: -; T2-LABEL: foo: define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { +; ARM-LABEL: foo: ; ARM: @ %bb.0: @ %entry ; ARM-NEXT: add r2, r0, #4 ; ARM-NEXT: mov r12, #1 @@ -44,6 +42,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; ARM-NEXT: add r0, r0, r1, lsl #2 ; ARM-NEXT: mov pc, lr ; +; THUMB-LABEL: foo: ; THUMB: @ %bb.0: @ %entry ; THUMB-NEXT: .save {r4, r5, r7, lr} ; THUMB-NEXT: push {r4, r5, r7, lr} @@ -91,6 +90,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; THUMB-NEXT: pop {r0} ; THUMB-NEXT: bx r0 ; +; T2-LABEL: foo: ; T2: @ %bb.0: @ %entry ; T2-NEXT: adds r2, r0, #4 ; T2-NEXT: mov.w r12, #1 @@ -125,6 +125,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; T2-NEXT: add.w r0, r0, r1, lsl #2 ; T2-NEXT: bx lr ; +; V8-LABEL: foo: ; V8: @ %bb.0: @ %entry ; V8-NEXT: adds r2, r0, #4 ; V8-NEXT: mov.w r12, #1 @@ -210,11 +211,8 @@ sw.epilog: ; preds = %tailrecurse.switch %struct.S = type { ptr, [1 x i8] } -; ARM-LABEL: bar: -; THUMB-LABEL: bar: -; T2-LABEL: bar: -; V8-LABEL: bar: define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { +; ARM-LABEL: bar: ; ARM: @ %bb.0: @ %entry ; ARM-NEXT: ldrb r2, [r0, #4] ; ARM-NEXT: ands r2, r2, #112 @@ -230,6 +228,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; ARM-NEXT: mov r0, #1 ; ARM-NEXT: mov pc, lr ; +; THUMB-LABEL: bar: ; THUMB: @ %bb.0: @ %entry ; THUMB-NEXT: ldrb r2, [r0, #4] ; THUMB-NEXT: movs r3, #112 @@ -253,6 +252,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; THUMB-NEXT: ands r0, r1 ; THUMB-NEXT: bx lr ; +; T2-LABEL: bar: ; T2: @ %bb.0: @ %entry ; T2-NEXT: ldrb r2, [r0, #4] ; T2-NEXT: ands r2, r2, #112 @@ -270,6 +270,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; T2-NEXT: movs r0, #1 ; T2-NEXT: bx lr ; +; V8-LABEL: bar: ; V8: @ %bb.0: @ %entry ; V8-NEXT: ldrb r2, [r0, #4] ; V8-NEXT: ands r2, r2, #112 diff --git a/llvm/test/CodeGen/ARM/select.ll b/llvm/test/CodeGen/ARM/select.ll index 4bb79651f040..24ca9aeac7f2 100644 --- a/llvm/test/CodeGen/ARM/select.ll +++ b/llvm/test/CodeGen/ARM/select.ll @@ -1,14 +1,25 @@ -; RUN: llc -mtriple=arm-apple-darwin %s -o - | FileCheck %s - -; RUN: llc -mtriple=arm-eabi -mattr=+vfp2 %s -o - \ -; RUN: | FileCheck %s --check-prefix=CHECK-VFP - -; RUN: llc -mtriple=thumbv7-apple-darwin -mattr=+neon,+thumb2 %s -o - \ -; RUN: | FileCheck %s --check-prefix=CHECK-NEON +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=armv7-eabi -mattr=-fpregs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ARM +; RUN: llc -mtriple=armv7-eabi -mattr=+vfp2 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-VFP +; RUN: llc -mtriple=thumbv7-apple-darwin -mattr=+neon,+thumb2 %s -o - | FileCheck %s --check-prefix=CHECK-NEON define i32 @f1(i32 %a.s) { -;CHECK-LABEL: f1: -;CHECK: moveq +; CHECK-LABEL: f1: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r1, #3 +; CHECK-NEXT: cmp r0, #4 +; CHECK-NEXT: movweq r1, #2 +; CHECK-NEXT: mov r0, r1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f1: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r1, #3 +; CHECK-NEON-NEXT: cmp r0, #4 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: moveq r1, #2 +; CHECK-NEON-NEXT: mov r0, r1 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp eq i32 %a.s, 4 %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -16,8 +27,22 @@ entry: } define i32 @f2(i32 %a.s) { -;CHECK-LABEL: f2: -;CHECK: movgt +; CHECK-LABEL: f2: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r1, #3 +; CHECK-NEXT: cmp r0, #4 +; CHECK-NEXT: movwgt r1, #2 +; CHECK-NEXT: mov r0, r1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f2: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r1, #3 +; CHECK-NEON-NEXT: cmp r0, #4 +; CHECK-NEON-NEXT: it gt +; CHECK-NEON-NEXT: movgt r1, #2 +; CHECK-NEON-NEXT: mov r0, r1 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp sgt i32 %a.s, 4 %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -25,8 +50,22 @@ entry: } define i32 @f3(i32 %a.s, i32 %b.s) { -;CHECK-LABEL: f3: -;CHECK: movlt +; CHECK-LABEL: f3: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwlt r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f3: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it lt +; CHECK-NEON-NEXT: movlt r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp slt i32 %a.s, %b.s %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -34,8 +73,22 @@ entry: } define i32 @f4(i32 %a.s, i32 %b.s) { -;CHECK-LABEL: f4: -;CHECK: movle +; CHECK-LABEL: f4: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwle r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f4: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it le +; CHECK-NEON-NEXT: movle r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp sle i32 %a.s, %b.s %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -43,8 +96,22 @@ entry: } define i32 @f5(i32 %a.u, i32 %b.u) { -;CHECK-LABEL: f5: -;CHECK: movls +; CHECK-LABEL: f5: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwls r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f5: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it ls +; CHECK-NEON-NEXT: movls r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp ule i32 %a.u, %b.u %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -52,8 +119,22 @@ entry: } define i32 @f6(i32 %a.u, i32 %b.u) { -;CHECK-LABEL: f6: -;CHECK: movhi +; CHECK-LABEL: f6: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwhi r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f6: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it hi +; CHECK-NEON-NEXT: movhi r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp ugt i32 %a.u, %b.u %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -61,11 +142,61 @@ entry: } define double @f7(double %a, double %b) { -;CHECK-LABEL: f7: -;CHECK: movmi -;CHECK: movpl -;CHECK-VFP-LABEL: f7: -;CHECK-VFP: vmovmi +; CHECK-ARM-LABEL: f7: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: .save {r4, r5, r11, lr} +; CHECK-ARM-NEXT: push {r4, r5, r11, lr} +; CHECK-ARM-NEXT: mov r4, r3 +; CHECK-ARM-NEXT: movw r3, #48758 +; CHECK-ARM-NEXT: mov r5, r2 +; CHECK-ARM-NEXT: movw r2, #14680 +; CHECK-ARM-NEXT: movt r2, #51380 +; CHECK-ARM-NEXT: movt r3, #16371 +; CHECK-ARM-NEXT: bl __aeabi_dcmplt +; CHECK-ARM-NEXT: cmp r0, #0 +; CHECK-ARM-NEXT: movwne r4, #0 +; CHECK-ARM-NEXT: movwne r5, #0 +; CHECK-ARM-NEXT: movtne r4, #49136 +; CHECK-ARM-NEXT: mov r0, r5 +; CHECK-ARM-NEXT: mov r1, r4 +; CHECK-ARM-NEXT: pop {r4, r5, r11, pc} +; +; CHECK-VFP-LABEL: f7: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vldr d17, .LCPI6_0 +; CHECK-VFP-NEXT: vmov d19, r0, r1 +; CHECK-VFP-NEXT: vmov.f64 d16, #-1.000000e+00 +; CHECK-VFP-NEXT: vcmp.f64 d19, d17 +; CHECK-VFP-NEXT: vmrs APSR_nzcv, fpscr +; CHECK-VFP-NEXT: vmov d18, r2, r3 +; CHECK-VFP-NEXT: vmovmi.f64 d18, d16 +; CHECK-VFP-NEXT: vmov r0, r1, d18 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 3 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI6_0: +; CHECK-VFP-NEXT: .long 3367254360 @ double 1.234 +; CHECK-VFP-NEXT: .long 1072938614 +; +; CHECK-NEON-LABEL: f7: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr d17, LCPI6_0 +; CHECK-NEON-NEXT: vmov d19, r0, r1 +; CHECK-NEON-NEXT: vmov d18, r2, r3 +; CHECK-NEON-NEXT: vcmp.f64 d19, d17 +; CHECK-NEON-NEXT: vmov.f64 d16, #-1.000000e+00 +; CHECK-NEON-NEXT: vmrs APSR_nzcv, fpscr +; CHECK-NEON-NEXT: it mi +; CHECK-NEON-NEXT: vmovmi.f64 d18, d16 +; CHECK-NEON-NEXT: vmov r0, r1, d18 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 3 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI6_0: +; CHECK-NEON-NEXT: .long 3367254360 @ double 1.234 +; CHECK-NEON-NEXT: .long 1072938614 +; CHECK-NEON-NEXT: .end_data_region %tmp = fcmp olt double %a, 1.234e+00 %tmp1 = select i1 %tmp, double -1.000e+00, double %b ret double %tmp1 @@ -77,18 +208,49 @@ define double @f7(double %a, double %b) { ; a lack of a custom lowering routine for an ISD::SELECT. This would result in ; two "it" blocks in the code: one for the "icmp" and another to move the index ; into the constant pool based on the value of the "icmp". If we have one "it" -; block generated, odds are good that we have close to the ideal code for this: +; block generated, odds are good that we have close to the ideal code for this. +define arm_apcscc float @f8(i32 %a) nounwind { +; CHECK-ARM-LABEL: f8: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: movw r1, #29905 +; CHECK-ARM-NEXT: movw r2, #1123 +; CHECK-ARM-NEXT: movt r1, #16408 +; CHECK-ARM-NEXT: cmp r0, r2 +; CHECK-ARM-NEXT: movweq r1, #62390 +; CHECK-ARM-NEXT: movteq r1, #16285 +; CHECK-ARM-NEXT: mov r0, r1 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f8: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: movw r2, #1123 +; CHECK-VFP-NEXT: adr r1, .LCPI7_0 +; CHECK-VFP-NEXT: cmp r0, r2 +; CHECK-VFP-NEXT: addeq r1, r1, #4 +; CHECK-VFP-NEXT: ldr r0, [r1] +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI7_0: +; CHECK-VFP-NEXT: .long 0x401874d1 @ float 2.38212991 +; CHECK-VFP-NEXT: .long 0x3f9df3b6 @ float 1.23399997 ; ; CHECK-NEON-LABEL: f8: -; CHECK-NEON: adr [[R2:r[0-9]+]], LCPI7_0 -; CHECK-NEON: movw [[R3:r[0-9]+]], #1123 -; CHECK-NEON-NEXT: cmp r0, [[R3]] -; CHECK-NEON-NEXT: it eq -; CHECK-NEON-NEXT: addeq{{.*}} [[R2]], #4 -; CHECK-NEON-NEXT: ldr -; CHECK-NEON: bx - -define arm_apcscc float @f8(i32 %a) nounwind { +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: adr r1, LCPI7_0 +; CHECK-NEON-NEXT: movw r2, #1123 +; CHECK-NEON-NEXT: cmp r0, r2 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: addeq r1, #4 +; CHECK-NEON-NEXT: ldr r0, [r1] +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI7_0: +; CHECK-NEON-NEXT: .long 0x401874d1 @ float 2.38212991 +; CHECK-NEON-NEXT: .long 0x3f9df3b6 @ float 1.23399997 +; CHECK-NEON-NEXT: .end_data_region %tmp = icmp eq i32 %a, 1123 %tmp1 = select i1 %tmp, float 0x3FF3BE76C0000000, float 0x40030E9A20000000 ret float %tmp1 @@ -98,10 +260,40 @@ define arm_apcscc float @f8(i32 %a) nounwind { ; Glue values can only have a single use, but the following test exposed a ; case where a SELECT was lowered with 2 uses of a comparison, causing the ; scheduler to assert. -; CHECK-VFP-LABEL: f9: - declare ptr @objc_msgSend(ptr, ptr, ...) define void @f9() optsize { +; CHECK-LABEL: f9: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r11, lr} +; CHECK-NEXT: push {r11, lr} +; CHECK-NEXT: .pad #8 +; CHECK-NEXT: sub sp, sp, #8 +; CHECK-NEXT: movw r2, #0 +; CHECK-NEXT: movw r3, #0 +; CHECK-NEXT: mov r1, #1065353216 +; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: movt r2, #16672 +; CHECK-NEXT: movt r3, #32704 +; CHECK-NEXT: strd r0, r1, [sp] +; CHECK-NEXT: bl objc_msgSend +; CHECK-NEXT: add sp, sp, #8 +; CHECK-NEXT: pop {r11, pc} +; +; CHECK-NEON-LABEL: f9: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: str lr, [sp, #-4]! +; CHECK-NEON-NEXT: sub sp, #8 +; CHECK-NEON-NEXT: movs r2, #0 +; CHECK-NEON-NEXT: movs r3, #0 +; CHECK-NEON-NEXT: mov.w r0, #1065353216 +; CHECK-NEON-NEXT: movs r1, #0 +; CHECK-NEON-NEXT: movt r2, #16672 +; CHECK-NEON-NEXT: movt r3, #32704 +; CHECK-NEON-NEXT: strd r1, r0, [sp] +; CHECK-NEON-NEXT: bl _objc_msgSend +; CHECK-NEON-NEXT: add sp, #8 +; CHECK-NEON-NEXT: ldr lr, [sp], #4 +; CHECK-NEON-NEXT: bx lr entry: %cmp = icmp eq ptr undef, inttoptr (i32 4 to ptr) %conv191 = select i1 %cmp, float -3.000000e+00, float 0.000000e+00 @@ -117,36 +309,151 @@ entry: ret void } -; CHECK-LABEL: f10: define float @f10(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatsisf +; CHECK-ARM-LABEL: f10: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: moveq r2, #1065353216 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f10: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI9_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI9_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f10: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI9_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI9_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = zext i1 %1 to i32 %3 = sitofp i32 %2 to float ret float %3 } -; CHECK-LABEL: f11: define float @f11(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatsisf +; CHECK-ARM-LABEL: f11: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: movweq r2, #0 +; CHECK-ARM-NEXT: movteq r2, #49024 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f11: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #-1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI10_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI10_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f11: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI10_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #-1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI10_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = sitofp i1 %1 to float ret float %2 } -; CHECK-LABEL: f12: define float @f12(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatunsisf +; CHECK-ARM-LABEL: f12: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: moveq r2, #1065353216 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f12: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI11_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI11_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f12: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI11_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI11_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = uitofp i1 %1 to float ret float %2 } -; CHECK-LABEL: test_overflow_recombine: define i1 @test_overflow_recombine(i32 %in1, i32 %in2) { -; CHECK: smull [[LO:r[0-9]+]], [[HI:r[0-9]+]] -; CHECK: subs [[ZERO:r[0-9]+]], [[HI]], [[LO]], asr #31 -; CHECK: movne [[ZERO]], #1 +; CHECK-LABEL: test_overflow_recombine: +; CHECK: @ %bb.0: +; CHECK-NEXT: mul r2, r0, r1 +; CHECK-NEXT: smmul r0, r0, r1 +; CHECK-NEXT: subs r0, r0, r2, asr #31 +; CHECK-NEXT: movwne r0, #1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: test_overflow_recombine: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: mul r2, r0, r1 +; CHECK-NEON-NEXT: smmul r0, r0, r1 +; CHECK-NEON-NEXT: subs.w r0, r0, r2, asr #31 +; CHECK-NEON-NEXT: it ne +; CHECK-NEON-NEXT: movne r0, #1 +; CHECK-NEON-NEXT: bx lr %prod = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %in1, i32 %in2) %overflow = extractvalue { i32, i1 } %prod, 1 ret i1 %overflow -- GitLab From 2bfa7d0e1691dcff095c602a0387a6d13213dc91 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Thu, 21 Mar 2024 21:48:10 +0800 Subject: [PATCH 048/933] [InstCombine] Fold `fmul X, -0.0` into `copysign(0.0, -X)` (#85772) `fneg + copysign` is better than fmul for analysis/codegen. godbolt: https://godbolt.org/z/eEs6dGd1G Alive2: https://alive2.llvm.org/ce/z/K3M5BA --- .../InstCombine/InstCombineMulDivRem.cpp | 15 ++- .../Transforms/InstCombine/binop-itofp.ll | 12 ++- llvm/test/Transforms/InstCombine/fmul.ll | 99 ++++++++++++++++++- llvm/test/Transforms/InstCombine/fpcast.ll | 5 +- 4 files changed, 116 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 9d4c271f990d..6e05fd8fb4d6 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -814,8 +814,19 @@ Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) { if (match(Op1, m_SpecificFP(-1.0))) return UnaryOperator::CreateFNegFMF(Op0, &I); - // With no-nans: X * 0.0 --> copysign(0.0, X) - if (I.hasNoNaNs() && match(Op1, m_PosZeroFP())) { + // With no-nans/no-infs: + // X * 0.0 --> copysign(0.0, X) + // X * -0.0 --> copysign(0.0, -X) + const APFloat *FPC; + if (match(Op1, m_APFloatAllowUndef(FPC)) && FPC->isZero() && + ((I.hasNoInfs() && + isKnownNeverNaN(Op0, /*Depth=*/0, SQ.getWithInstruction(&I))) || + isKnownNeverNaN(&I, /*Depth=*/0, SQ.getWithInstruction(&I)))) { + if (FPC->isNegative()) + Op0 = Builder.CreateFNegFMF(Op0, &I); + Op1 = Constant::replaceUndefsWith( + cast(Op1), + ConstantFP::get(Op1->getType()->getScalarType(), *FPC)); CallInst *CopySign = Builder.CreateIntrinsic(Intrinsic::copysign, {I.getType()}, {Op1, Op0}, &I); return replaceInstUsesWith(I, CopySign); diff --git a/llvm/test/Transforms/InstCombine/binop-itofp.ll b/llvm/test/Transforms/InstCombine/binop-itofp.ll index 82cdb3ce6bee..cd9ec1e59203 100644 --- a/llvm/test/Transforms/InstCombine/binop-itofp.ll +++ b/llvm/test/Transforms/InstCombine/binop-itofp.ll @@ -1012,7 +1012,7 @@ define float @missed_nonzero_check_on_constant_for_si_fmul(i1 %c, i1 %.b, ptr %g ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i32 65529, i32 53264 ; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[SEL]] to i16 ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp i16 [[CONV_I]] to float -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul float [[CONV1_I]], 0.000000e+00 +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret float [[MUL3_I_I]] ; @@ -1031,7 +1031,7 @@ define <2 x float> @missed_nonzero_check_on_constant_for_si_fmul_vec(i1 %c, i1 % ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], zeroinitializer +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; @@ -1050,7 +1050,8 @@ define float @negzero_check_on_constant_for_si_fmul(i1 %c, i1 %.b, ptr %g_2345) ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i32 65529, i32 53264 ; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[SEL]] to i16 ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp i16 [[CONV_I]] to float -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul float [[CONV1_I]], -0.000000e+00 +; CHECK-NEXT: [[TMP1:%.*]] = fneg float [[CONV1_I]] +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[TMP1]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret float [[MUL3_I_I]] ; @@ -1069,7 +1070,7 @@ define <2 x float> @nonzero_check_on_constant_for_si_fmul_vec_w_undef(i1 %c, i1 ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; @@ -1111,7 +1112,8 @@ define <2 x float> @nonzero_check_on_constant_for_si_fmul_negz_vec_w_undef(i1 %c ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], +; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x float> [[CONV1_I]] +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[TMP1]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll index 96e57939d285..f6435f003289 100644 --- a/llvm/test/Transforms/InstCombine/fmul.ll +++ b/llvm/test/Transforms/InstCombine/fmul.ll @@ -1250,7 +1250,7 @@ define half @mul_zero_nnan(half %x) { define <2 x float> @mul_zero_nnan_vec_poison(<2 x float> %x) { ; CHECK-LABEL: @mul_zero_nnan_vec_poison( -; CHECK-NEXT: [[R:%.*]] = call nnan <2 x float> @llvm.copysign.v2f32(<2 x float> , <2 x float> [[X:%.*]]) +; CHECK-NEXT: [[R:%.*]] = call nnan <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[X:%.*]]) ; CHECK-NEXT: ret <2 x float> [[R]] ; %r = fmul nnan <2 x float> %x, @@ -1268,13 +1268,104 @@ define half @mul_zero(half %x) { ret half %r } -; TODO: This could be fneg+copysign. - define half @mul_negzero_nnan(half %x) { ; CHECK-LABEL: @mul_negzero_nnan( -; CHECK-NEXT: [[R:%.*]] = fmul nnan half [[X:%.*]], 0xH8000 +; CHECK-NEXT: [[TMP1:%.*]] = fneg nnan half [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = call nnan half @llvm.copysign.f16(half 0xH0000, half [[TMP1]]) ; CHECK-NEXT: ret half [[R]] ; %r = fmul nnan half %x, -0.0 ret half %r } + +define float @mul_pos_zero_nnan_ninf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan_ninf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[A:%.*]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_pos_zero_nnan(float nofpclass(nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = fmul float [[A:%.*]], 0.000000e+00 +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_pos_zero_nnan_ninf_fmf(float nofpclass(nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan_ninf_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = call ninf float @llvm.copysign.f32(float 0.000000e+00, float [[A:%.*]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul ninf float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_ninf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, -0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_fmf(float %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg nnan float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call nnan float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul nnan float %a, -0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_ninf_fmf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg nnan ninf float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call nnan ninf float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul nnan ninf float %a, -0.000000e+00 + ret float %ret +} + +define <3 x float> @mul_neg_zero_nnan_ninf_vec(<3 x float> nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf_vec( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg <3 x float> [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call <3 x float> @llvm.copysign.v3f32(<3 x float> zeroinitializer, <3 x float> [[TMP0]]) +; CHECK-NEXT: ret <3 x float> [[RET]] +; +entry: + %ret = fmul <3 x float> %a, + ret <3 x float> %ret +} + +define <3 x float> @mul_mixed_zero_nnan_ninf_vec(<3 x float> nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_mixed_zero_nnan_ninf_vec( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = fmul <3 x float> [[A:%.*]], +; CHECK-NEXT: ret <3 x float> [[RET]] +; +entry: + %ret = fmul <3 x float> %a, + ret <3 x float> %ret +} diff --git a/llvm/test/Transforms/InstCombine/fpcast.ll b/llvm/test/Transforms/InstCombine/fpcast.ll index 32bfdb52bb5f..ac4b88fcddd7 100644 --- a/llvm/test/Transforms/InstCombine/fpcast.ll +++ b/llvm/test/Transforms/InstCombine/fpcast.ll @@ -424,10 +424,7 @@ define i32 @fptosi_select(i1 %cond) { define i32 @mul_pos_zero_convert(i32 %a) { ; CHECK-LABEL: @mul_pos_zero_convert( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[FP:%.*]] = sitofp i32 [[A:%.*]] to float -; CHECK-NEXT: [[RET:%.*]] = fmul float [[FP]], 0.000000e+00 -; CHECK-NEXT: [[CONV:%.*]] = fptosi float [[RET]] to i32 -; CHECK-NEXT: ret i32 [[CONV]] +; CHECK-NEXT: ret i32 0 ; entry: %fp = sitofp i32 %a to float -- GitLab From 857161c367a1cdca926dbe0d2601e3afc52f03f9 Mon Sep 17 00:00:00 2001 From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:57:10 +0000 Subject: [PATCH 049/933] [AMDGPU] MCExpr-ify MC layer kernel descriptor (#80855) Kernel descriptor attributes, with their respective emit and asm parse functionality, converted to MCExpr. --- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 40 +- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 11 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 193 ++++++--- .../MCTargetDesc/AMDGPUMCKernelDescriptor.cpp | 32 ++ .../MCTargetDesc/AMDGPUMCKernelDescriptor.h | 51 +++ .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 404 +++++++++++------- .../MCTargetDesc/AMDGPUTargetStreamer.h | 33 +- .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 + .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 88 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 10 +- llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s | 27 ++ llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s | 281 ++++++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s | 190 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s | 186 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s | 184 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s | 168 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s | 171 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s | 148 +++++++ llvm/test/MC/AMDGPU/hsa-tg-split.s | 74 ++++ 19 files changed, 1988 insertions(+), 304 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h create mode 100644 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s create mode 100644 llvm/test/MC/AMDGPU/hsa-tg-split.s diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 72e8b59e0a40..052b231d62a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -22,6 +22,7 @@ #include "AMDKernelCodeT.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "R600AsmPrinter.h" #include "SIMachineFunctionInfo.h" @@ -428,38 +429,43 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( return KernelCodeProperties; } -amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor( - const MachineFunction &MF, - const SIProgramInfo &PI) const { +MCKernelDescriptor +AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF, + const SIProgramInfo &PI) const { const GCNSubtarget &STM = MF.getSubtarget(); const Function &F = MF.getFunction(); const SIMachineFunctionInfo *Info = MF.getInfo(); + MCContext &Ctx = MF.getContext(); - amdhsa::kernel_descriptor_t KernelDescriptor; - memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor)); + MCKernelDescriptor KernelDescriptor; assert(isUInt<32>(PI.ScratchSize)); assert(isUInt<32>(PI.getComputePGMRSrc1(STM))); assert(isUInt<32>(PI.getComputePGMRSrc2())); - KernelDescriptor.group_segment_fixed_size = PI.LDSSize; - KernelDescriptor.private_segment_fixed_size = PI.ScratchSize; + KernelDescriptor.group_segment_fixed_size = + MCConstantExpr::create(PI.LDSSize, Ctx); + KernelDescriptor.private_segment_fixed_size = + MCConstantExpr::create(PI.ScratchSize, Ctx); Align MaxKernArgAlign; - KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign); + KernelDescriptor.kernarg_size = MCConstantExpr::create( + STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx); - KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM); - KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(); - KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF); + KernelDescriptor.compute_pgm_rsrc1 = + MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx); + KernelDescriptor.compute_pgm_rsrc2 = + MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx); + KernelDescriptor.kernel_code_properties = + MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx); assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0); - if (STM.hasGFX90AInsts()) - KernelDescriptor.compute_pgm_rsrc3 = - CurrentProgramInfo.ComputePGMRSrc3GFX90A; + KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create( + STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx); - if (AMDGPU::hasKernargPreload(STM)) - KernelDescriptor.kernarg_preload = - static_cast(Info->getNumKernargPreloadedSGPRs()); + KernelDescriptor.kernarg_preload = MCConstantExpr::create( + AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0, + Ctx); return KernelDescriptor; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h index 79326cd3d328..b8b2718d293e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h @@ -28,15 +28,12 @@ class MCCodeEmitter; class MCOperand; namespace AMDGPU { +struct MCKernelDescriptor; namespace HSAMD { class MetadataStreamer; } } // namespace AMDGPU -namespace amdhsa { -struct kernel_descriptor_t; -} - class AMDGPUAsmPrinter final : public AsmPrinter { private: unsigned CodeObjectVersion; @@ -75,9 +72,9 @@ private: uint16_t getAmdhsaKernelCodeProperties( const MachineFunction &MF) const; - amdhsa::kernel_descriptor_t getAmdhsaKernelDescriptor( - const MachineFunction &MF, - const SIProgramInfo &PI) const; + AMDGPU::MCKernelDescriptor + getAmdhsaKernelDescriptor(const MachineFunction &MF, + const SIProgramInfo &PI) const; void initTargetStreamer(Module &M); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..38850f5acadd 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -8,6 +8,7 @@ #include "AMDKernelCodeT.h" #include "MCTargetDesc/AMDGPUMCExpr.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -5417,7 +5418,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (getParser().parseIdentifier(KernelName)) return true; - kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI()); + AMDGPU::MCKernelDescriptor KD = + getDefaultAmdhsaKernelDescriptor(&getSTI(), getContext()); StringSet<> Seen; @@ -5457,89 +5459,111 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return TokError(".amdhsa_ directives cannot be repeated"); SMLoc ValStart = getLoc(); - int64_t IVal; - if (getParser().parseAbsoluteExpression(IVal)) + const MCExpr *ExprVal; + if (getParser().parseExpression(ExprVal)) return true; SMLoc ValEnd = getLoc(); SMRange ValRange = SMRange(ValStart, ValEnd); - if (IVal < 0) - return OutOfRangeError(ValRange); - + int64_t IVal = 0; uint64_t Val = IVal; + bool EvaluatableExpr; + if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) { + if (IVal < 0) + return OutOfRangeError(ValRange); + Val = IVal; + } #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \ - if (!isUInt(VALUE)) \ + if (!isUInt(Val)) \ return OutOfRangeError(RANGE); \ - AMDHSA_BITS_SET(FIELD, ENTRY, VALUE); + AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \ + getContext()); + +// Some fields use the parsed value immediately which requires the expression to +// be solvable. +#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \ + if (!(RESOLVED)) \ + return Error(IDRange.Start, "directive should have resolvable expression", \ + IDRange); if (ID == ".amdhsa_group_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.group_segment_fixed_size = Val; + KD.group_segment_fixed_size = ExprVal; } else if (ID == ".amdhsa_private_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.private_segment_fixed_size = Val; + KD.private_segment_fixed_size = ExprVal; } else if (ID == ".amdhsa_kernarg_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.kernarg_size = Val; + KD.kernarg_size = ExprVal; } else if (ID == ".amdhsa_user_sgpr_count") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); ExplicitUserSGPRCount = Val; } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 4; } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val > getMaxNumUserSGPRs()) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal, ValRange); if (Val) { ImpliedUserSGPRCount += Val; PreloadLength = Val; } } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val >= 1024) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal, ValRange); if (Val) PreloadOffset = Val; } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_queue_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_dispatch_id") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; @@ -5548,34 +5572,39 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val, - ValRange); + KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_private_segment_size") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 1; } else if (ID == ".amdhsa_wavefront_size32") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); EnableWavefrontSize32 = Val; PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - Val, ValRange); + KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal, + ValRange); } else if (ID == ".amdhsa_uses_dynamic_stack") { PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, Val, ValRange); + KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal, + ValRange); } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") { if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, + ValRange); } else if (ID == ".amdhsa_enable_private_segment") { if (!hasArchitectedFlatScratch()) return Error( @@ -5583,42 +5612,48 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "directive is not supported without architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, + ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_info") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal, ValRange); } else if (ID == ".amdhsa_system_vgpr_workitem_id") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val, + COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal, ValRange); } else if (ID == ".amdhsa_next_free_vgpr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); VGPRRange = ValRange; NextFreeVGPR = Val; } else if (ID == ".amdhsa_next_free_sgpr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); SGPRRange = ValRange; NextFreeSGPR = Val; } else if (ID == ".amdhsa_accum_offset") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); AccumOffset = Val; } else if (ID == ".amdhsa_reserve_vcc") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!isUInt<1>(Val)) return OutOfRangeError(ValRange); ReserveVCC = Val; } else if (ID == ".amdhsa_reserve_flat_scratch") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 7) return Error(IDRange.Start, "directive requires gfx7+", IDRange); if (hasArchitectedFlatScratch()) @@ -5638,97 +5673,105 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { IDRange); } else if (ID == ".amdhsa_float_round_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_round_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_denorm_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_denorm_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val, + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal, ValRange); } else if (ID == ".amdhsa_dx10_clamp") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal, ValRange); } else if (ID == ".amdhsa_ieee_mode") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal, ValRange); } else if (ID == ".amdhsa_fp16_overflow") { if (IVersion.Major < 9) return Error(IDRange.Start, "directive requires gfx9+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal, ValRange); } else if (ID == ".amdhsa_tg_split") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val, - ValRange); + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, + ExprVal, ValRange); } else if (ID == ".amdhsa_workgroup_processor_mode") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal, ValRange); } else if (ID == ".amdhsa_memory_ordered") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal, ValRange); } else if (ID == ".amdhsa_forward_progress") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal, ValRange); } else if (ID == ".amdhsa_shared_vgpr_count") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10 || IVersion.Major >= 12) return Error(IDRange.Start, "directive requires gfx10 or gfx11", IDRange); SharedVGPRCount = Val; PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, - COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, Val, + COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_denorm_src") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_overflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_underflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_inexact") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_int_div_zero") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_round_robin_scheduling") { if (IVersion.Major < 12) return Error(IDRange.Start, "directive requires gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val, + COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal, ValRange); } else { return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange); @@ -5755,15 +5798,18 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt( VGPRBlocks)) return OutOfRangeError(VGPRRange); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks); + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, MCConstantExpr::create(VGPRBlocks, getContext()), + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, getContext()); if (!isUInt( SGPRBlocks)) return OutOfRangeError(SGPRRange); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, - SGPRBlocks); + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, MCConstantExpr::create(SGPRBlocks, getContext()), + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, getContext()); if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount) return TokError("amdgpu_user_sgpr_count smaller than than implied by " @@ -5774,11 +5820,17 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt(UserSGPRCount)) return TokError("too many user SGPRs enabled"); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, - UserSGPRCount); - - if (PreloadLength && KD.kernarg_size && - (PreloadLength * 4 + PreloadOffset * 4 > KD.kernarg_size)) + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()), + COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext()); + + int64_t IVal = 0; + if (!KD.kernarg_size->evaluateAsAbsolute(IVal)) + return TokError("Kernarg size should be resolvable"); + uint64_t kernarg_size = IVal; + if (PreloadLength && kernarg_size && + (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size)) return TokError("Kernarg preload length + offset is larger than the " "kernarg segment size"); @@ -5790,8 +5842,11 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "increments of 4"); if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4)) return TokError("accum_offset exceeds total VGPR allocation"); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, - (AccumOffset / 4 - 1)); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc3, + MCConstantExpr::create(AccumOffset / 4 - 1, getContext()), + COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, + COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); } if (IVersion.Major >= 10 && IVersion.Major < 12) { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp new file mode 100644 index 000000000000..0179d575464d --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp @@ -0,0 +1,32 @@ +//===--- AMDHSAKernelDescriptor.h -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMCKernelDescriptor.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCExpr.h" + +using namespace llvm; +using namespace llvm::AMDGPU; + +void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value, + uint32_t Shift, uint32_t Mask, + MCContext &Ctx) { + auto Sft = MCConstantExpr::create(Shift, Ctx); + auto Msk = MCConstantExpr::create(Mask, Ctx); + Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx); + Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx), + Ctx); +} + +const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift, + uint32_t Mask, MCContext &Ctx) { + auto Sft = MCConstantExpr::create(Shift, Ctx); + auto Msk = MCConstantExpr::create(Mask, Ctx); + return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft, + Ctx); +} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h new file mode 100644 index 000000000000..71659e642dd7 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h @@ -0,0 +1,51 @@ +//===--- AMDGPUMCKernelDescriptor.h ---------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// AMDHSA kernel descriptor MCExpr struct for use in MC layer. Uses +/// AMDHSAKernelDescriptor.h for sizes and constants. +/// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H +#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H + +#include "llvm/Support/AMDHSAKernelDescriptor.h" + +namespace llvm { +class MCExpr; +class MCContext; +namespace AMDGPU { + +struct MCKernelDescriptor { + const MCExpr *group_segment_fixed_size = nullptr; + const MCExpr *private_segment_fixed_size = nullptr; + const MCExpr *kernarg_size = nullptr; + const MCExpr *compute_pgm_rsrc3 = nullptr; + const MCExpr *compute_pgm_rsrc1 = nullptr; + const MCExpr *compute_pgm_rsrc2 = nullptr; + const MCExpr *kernel_code_properties = nullptr; + const MCExpr *kernarg_preload = nullptr; + + // MCExpr for: + // Dst = Dst & ~Mask + // Dst = Dst | (Value << Shift) + static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, + uint32_t Mask, MCContext &Ctx); + + // MCExpr for: + // return (Src & Mask) >> Shift + static const MCExpr *bits_get(const MCExpr *Src, uint32_t Shift, + uint32_t Mask, MCContext &Ctx); +}; + +} // end namespace AMDGPU +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 4742b0b3e52e..3006fcdb9282 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "AMDGPUTargetStreamer.h" +#include "AMDGPUMCKernelDescriptor.h" #include "AMDGPUPTNote.h" #include "AMDKernelCodeT.h" #include "Utils/AMDGPUBaseInfo.h" @@ -307,94 +308,142 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR, + const MCKernelDescriptor &KD, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { IsaVersion IVersion = getIsaVersion(STI.getCPU()); + const MCAsmInfo *MAI = getContext().getAsmInfo(); OS << "\t.amdhsa_kernel " << KernelName << '\n'; -#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \ - STREAM << "\t\t" << DIRECTIVE << " " \ - << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n'; - - OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size - << '\n'; - OS << "\t\t.amdhsa_private_segment_fixed_size " - << KD.private_segment_fixed_size << '\n'; - OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n'; - - PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT); + auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask, + StringRef Directive) { + int64_t IVal; + OS << "\t\t" << Directive << ' '; + const MCExpr *pgm_rsrc1_bits = + MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext()); + if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal)) + OS << static_cast(IVal); + else + pgm_rsrc1_bits->print(OS, MAI); + OS << '\n'; + }; + + OS << "\t\t.amdhsa_group_segment_fixed_size "; + KD.group_segment_fixed_size->print(OS, MAI); + OS << '\n'; + + OS << "\t\t.amdhsa_private_segment_fixed_size "; + KD.private_segment_fixed_size->print(OS, MAI); + OS << '\n'; + + OS << "\t\t.amdhsa_kernarg_size "; + KD.kernarg_size->print(OS, MAI); + OS << '\n'; + + PrintField( + KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count"); if (!hasArchitectedFlatScratch(STI)) - PRINT_FIELD( - OS, ".amdhsa_user_sgpr_private_segment_buffer", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID); + PrintField( + KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, + ".amdhsa_user_sgpr_private_segment_buffer"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, + ".amdhsa_user_sgpr_dispatch_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, + ".amdhsa_user_sgpr_queue_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, + ".amdhsa_user_sgpr_kernarg_segment_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, + ".amdhsa_user_sgpr_dispatch_id"); if (!hasArchitectedFlatScratch(STI)) - PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, + ".amdhsa_user_sgpr_flat_scratch_init"); if (hasKernargPreload(STI)) { - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD, - kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD, - kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET); + PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT, + amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, + ".amdhsa_user_sgpr_kernarg_preload_length"); + PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT, + amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, + ".amdhsa_user_sgpr_kernarg_preload_offset"); } - PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); + PrintField( + KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, + ".amdhsa_user_sgpr_private_segment_size"); if (IVersion.Major >= 10) - PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + ".amdhsa_wavefront_size32"); if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) - PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK); - PRINT_FIELD(OS, - (hasArchitectedFlatScratch(STI) - ? ".amdhsa_enable_private_segment" - : ".amdhsa_system_sgpr_private_segment_wavefront_offset"), - KD, compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO); - PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, + ".amdhsa_uses_dynamic_stack"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, + (hasArchitectedFlatScratch(STI) + ? ".amdhsa_enable_private_segment" + : ".amdhsa_system_sgpr_private_segment_wavefront_offset")); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, + ".amdhsa_system_sgpr_workgroup_id_x"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, + ".amdhsa_system_sgpr_workgroup_id_y"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, + ".amdhsa_system_sgpr_workgroup_id_z"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, + ".amdhsa_system_sgpr_workgroup_info"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, + ".amdhsa_system_vgpr_workitem_id"); // These directives are required. OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n'; OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n'; - if (AMDGPU::isGFX90A(STI)) - OS << "\t\t.amdhsa_accum_offset " << - (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4 - << '\n'; + if (AMDGPU::isGFX90A(STI)) { + // MCExpr equivalent of taking the (accum_offset + 1) * 4. + const MCExpr *accum_bits = MCKernelDescriptor::bits_get( + KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); + accum_bits = MCBinaryExpr::createAdd( + accum_bits, MCConstantExpr::create(1, getContext()), getContext()); + accum_bits = MCBinaryExpr::createMul( + accum_bits, MCConstantExpr::create(4, getContext()), getContext()); + OS << "\t\t.amdhsa_accum_offset "; + int64_t IVal; + if (accum_bits->evaluateAsAbsolute(IVal)) { + OS << static_cast(IVal); + } else { + accum_bits->print(OS, MAI); + } + OS << '\n'; + } if (!ReserveVCC) OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n'; @@ -411,74 +460,105 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( break; } - PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32); - PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64); - PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32); - PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, + ".amdhsa_float_round_mode_32"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, + ".amdhsa_float_round_mode_16_64"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, + ".amdhsa_float_denorm_mode_32"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, + ".amdhsa_float_denorm_mode_16_64"); if (IVersion.Major < 12) { - PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP); - PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, + ".amdhsa_dx10_clamp"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, + ".amdhsa_ieee_mode"); + } + if (IVersion.Major >= 9) { + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, + ".amdhsa_fp16_overflow"); } - if (IVersion.Major >= 9) - PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL); if (AMDGPU::isGFX90A(STI)) - PRINT_FIELD(OS, ".amdhsa_tg_split", KD, - compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT); + PrintField(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split"); if (IVersion.Major >= 10) { - PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE); - PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED); - PRINT_FIELD(OS, ".amdhsa_forward_progress", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, + ".amdhsa_workgroup_processor_mode"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, + ".amdhsa_memory_ordered"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, + ".amdhsa_forward_progress"); } if (IVersion.Major >= 10 && IVersion.Major < 12) { - PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT); + PrintField(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, + ".amdhsa_shared_vgpr_count"); } - if (IVersion.Major >= 12) - PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN); - PRINT_FIELD( - OS, ".amdhsa_exception_fp_ieee_invalid_op", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION); - PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE); - PRINT_FIELD( - OS, ".amdhsa_exception_fp_ieee_div_zero", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT); - PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO); -#undef PRINT_FIELD + if (IVersion.Major >= 12) { + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, + ".amdhsa_round_robin_scheduling"); + } + PrintField( + KD.compute_pgm_rsrc2, + amdhsa:: + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, + ".amdhsa_exception_fp_ieee_invalid_op"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, + ".amdhsa_exception_fp_denorm_src"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa:: + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, + ".amdhsa_exception_fp_ieee_div_zero"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, + ".amdhsa_exception_fp_ieee_overflow"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, + ".amdhsa_exception_fp_ieee_underflow"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, + ".amdhsa_exception_fp_ieee_inexact"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, + ".amdhsa_exception_int_div_zero"); OS << "\t.end_amdhsa_kernel\n"; } @@ -835,7 +915,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + const MCKernelDescriptor &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { auto &Streamer = getStreamer(); auto &Context = Streamer.getContext(); @@ -853,7 +933,7 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( // Kernel descriptor symbol's type and size are fixed. KernelDescriptorSymbol->setType(ELF::STT_OBJECT); KernelDescriptorSymbol->setSize( - MCConstantExpr::create(sizeof(KernelDescriptor), Context)); + MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context)); // The visibility of the kernel code symbol must be protected or less to allow // static relocations from the kernel descriptor to be used. @@ -861,31 +941,43 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED); Streamer.emitLabel(KernelDescriptorSymbol); - Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size); - Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size); - Streamer.emitInt32(KernelDescriptor.kernarg_size); - - for (uint8_t Res : KernelDescriptor.reserved0) - Streamer.emitInt8(Res); + Streamer.emitValue( + KernelDescriptor.group_segment_fixed_size, + sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size)); + Streamer.emitValue( + KernelDescriptor.private_segment_fixed_size, + sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size)); + Streamer.emitValue(KernelDescriptor.kernarg_size, + sizeof(amdhsa::kernel_descriptor_t::kernarg_size)); + + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i) + Streamer.emitInt8(0u); // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The // expression being created is: // (start of kernel code) - (start of kernel descriptor) // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64. - Streamer.emitValue(MCBinaryExpr::createSub( - MCSymbolRefExpr::create( - KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context), - MCSymbolRefExpr::create( - KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context), - Context), - sizeof(KernelDescriptor.kernel_code_entry_byte_offset)); - for (uint8_t Res : KernelDescriptor.reserved1) - Streamer.emitInt8(Res); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2); - Streamer.emitInt16(KernelDescriptor.kernel_code_properties); - Streamer.emitInt16(KernelDescriptor.kernarg_preload); - for (uint8_t Res : KernelDescriptor.reserved3) - Streamer.emitInt8(Res); + Streamer.emitValue( + MCBinaryExpr::createSub( + MCSymbolRefExpr::create(KernelCodeSymbol, + MCSymbolRefExpr::VK_AMDGPU_REL64, Context), + MCSymbolRefExpr::create(KernelDescriptorSymbol, + MCSymbolRefExpr::VK_None, Context), + Context), + sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset)); + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i) + Streamer.emitInt8(0u); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3)); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1)); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2)); + Streamer.emitValue( + KernelDescriptor.kernel_code_properties, + sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties)); + Streamer.emitValue(KernelDescriptor.kernarg_preload, + sizeof(amdhsa::kernel_descriptor_t::kernarg_preload)); + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i) + Streamer.emitInt8(0u); } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h index 5aa80ff578c6..706897a5dc1f 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h @@ -22,15 +22,13 @@ class MCSymbol; class formatted_raw_ostream; namespace AMDGPU { + +struct MCKernelDescriptor; namespace HSAMD { struct Metadata; } } // namespace AMDGPU -namespace amdhsa { -struct kernel_descriptor_t; -} - class AMDGPUTargetStreamer : public MCTargetStreamer { AMDGPUPALMetadata PALMetadata; @@ -94,10 +92,11 @@ public: return true; } - virtual void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {} + virtual void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) {} static StringRef getArchNameFromElfMach(unsigned ElfMach); static unsigned getElfMach(StringRef GPU); @@ -150,10 +149,11 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; + void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) override; }; class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer { @@ -205,10 +205,11 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; + void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) override; }; } #endif diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 0842a58f794b..14a02b6d8e36 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp + AMDGPUMCKernelDescriptor.cpp R600InstPrinter.cpp R600MCCodeEmitter.cpp R600MCTargetDesc.cpp diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..4970055c4bdb 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -10,6 +10,7 @@ #include "AMDGPU.h" #include "AMDGPUAsmUtils.h" #include "AMDKernelCodeT.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" @@ -20,6 +21,7 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1215,44 +1217,64 @@ void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, } } -amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( - const MCSubtargetInfo *STI) { +MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, + MCContext &Ctx) { IsaVersion Version = getIsaVersion(STI->getCPU()); - amdhsa::kernel_descriptor_t KD; - memset(&KD, 0, sizeof(KD)); - - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, - amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); - if (Version.Major >= 12) { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0); - } else { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1); + MCKernelDescriptor KD; + const MCExpr *ZeroMCExpr = MCConstantExpr::create(0, Ctx); + const MCExpr *OneMCExpr = MCConstantExpr::create(1, Ctx); + + KD.group_segment_fixed_size = ZeroMCExpr; + KD.private_segment_fixed_size = ZeroMCExpr; + KD.compute_pgm_rsrc1 = ZeroMCExpr; + KD.compute_pgm_rsrc2 = ZeroMCExpr; + KD.compute_pgm_rsrc3 = ZeroMCExpr; + KD.kernarg_size = ZeroMCExpr; + KD.kernel_code_properties = ZeroMCExpr; + KD.kernarg_preload = ZeroMCExpr; + + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, + MCConstantExpr::create(amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE, Ctx), + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Ctx); + if (Version.Major < 12) { + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Ctx); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Ctx); } - AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc2, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Ctx); if (Version.Major >= 10) { - AMDHSA_BITS_SET(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, - STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1); - } - if (AMDGPU::isGFX90A(*STI)) { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, - STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); + if (STI->getFeatureBits().test(FeatureWavefrontSize32)) + MCKernelDescriptor::bits_set( + KD.kernel_code_properties, OneMCExpr, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, Ctx); + if (!STI->getFeatureBits().test(FeatureCuMode)) + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Ctx); + + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Ctx); } + if (AMDGPU::isGFX90A(*STI) && STI->getFeatureBits().test(FeatureTgSplit)) + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc3, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx); return KD; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..32b73f1d868d 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -26,6 +26,7 @@ struct Align; class Argument; class Function; class GlobalValue; +class MCContext; class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; @@ -34,12 +35,9 @@ class StringRef; class Triple; class raw_ostream; -namespace amdhsa { -struct kernel_descriptor_t; -} - namespace AMDGPU { +struct MCKernelDescriptor; struct IsaVersion; /// Generic target versions emitted by this version of LLVM. @@ -852,8 +850,8 @@ unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc); void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, const MCSubtargetInfo *STI); -amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( - const MCSubtargetInfo *STI); +MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, + MCContext &Ctx); bool isGroupSegment(const GlobalValue *GV); bool isGlobalSegment(const GlobalValue *GV); diff --git a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s new file mode 100644 index 000000000000..4623500987be --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s @@ -0,0 +1,27 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// OBJDUMP: 0000 00000000 0f000000 00000000 00000000 + +.text + +.p2align 8 +.type caller,@function +caller: + s_endpgm + +.rodata + +.p2align 6 +.amdhsa_kernel caller + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) +.end_amdhsa_kernel + +.set callee1.private_seg_size, 4 +.set callee2.private_seg_size, 15 + +// ASM: .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) diff --git a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s new file mode 100644 index 000000000000..fab3e893352b --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s @@ -0,0 +1,281 @@ +// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a %s 2>&1 | FileCheck --check-prefix=ASM %s + +// Some expression currently require (immediately) solvable expressions, i.e., +// they don't depend on yet-unknown symbolic values. + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type user_sgpr_count,@function +user_sgpr_count: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_count + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_count defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_count + +.p2align 8 +.type user_sgpr_private_segment_buffer,@function +user_sgpr_private_segment_buffer: + s_endpgm + +.amdhsa_kernel user_sgpr_private_segment_buffer + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_private_segment_buffer defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer + +.p2align 8 +.type user_sgpr_kernarg_preload_length,@function +user_sgpr_kernarg_preload_length: + s_endpgm + +.amdhsa_kernel user_sgpr_kernarg_preload_length + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_preload_length defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length defined_boolean + +.p2align 8 +.type user_sgpr_kernarg_preload_offset,@function +user_sgpr_kernarg_preload_offset: + s_endpgm + +.amdhsa_kernel user_sgpr_kernarg_preload_offset + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean + +.p2align 8 +.type user_sgpr_dispatch_ptr,@function +user_sgpr_dispatch_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_dispatch_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_dispatch_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr + +.p2align 8 +.type user_sgpr_queue_ptr,@function +user_sgpr_queue_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_queue_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_queue_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr + +.p2align 8 +.type user_sgpr_kernarg_segment_ptr,@function +user_sgpr_kernarg_segment_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_kernarg_segment_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_segment_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr + +.p2align 8 +.type user_sgpr_dispatch_id,@function +user_sgpr_dispatch_id: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_dispatch_id + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_dispatch_id defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id + +.p2align 8 +.type user_sgpr_flat_scratch_init,@function +user_sgpr_flat_scratch_init: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_flat_scratch_init + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_flat_scratch_init defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init + +.p2align 8 +.type user_sgpr_private_segment_size,@function +user_sgpr_private_segment_size: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_private_segment_size + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_private_segment_size defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size + +.p2align 8 +.type wavefront_size32,@function +wavefront_size32: + s_endpgm + +.p2align 6 +.amdhsa_kernel wavefront_size32 + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_wavefront_size32 defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_wavefront_size32 + +.p2align 8 +.type next_free_vgpr,@function +next_free_vgpr: + s_endpgm + +.p2align 6 +.amdhsa_kernel next_free_vgpr + .amdhsa_next_free_vgpr defined_boolean + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_next_free_vgpr + +.p2align 8 +.type next_free_sgpr,@function +next_free_sgpr: + s_endpgm + +.p2align 6 +.amdhsa_kernel next_free_sgpr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr defined_boolean + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_next_free_sgpr + +.p2align 8 +.type accum_offset,@function +accum_offset: + s_endpgm + +.p2align 6 +.amdhsa_kernel accum_offset + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_accum_offset + +.p2align 8 +.type reserve_vcc,@function +reserve_vcc: + s_endpgm + +.p2align 6 +.amdhsa_kernel reserve_vcc + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_reserve_vcc defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_reserve_vcc + +.p2align 8 +.type reserve_flat_scratch,@function +reserve_flat_scratch: + s_endpgm + +.p2align 6 +.amdhsa_kernel reserve_flat_scratch + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_reserve_flat_scratch defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_reserve_flat_scratch + +.p2align 8 +.type shared_vgpr_count,@function +shared_vgpr_count: + s_endpgm + +.p2align 6 +.amdhsa_kernel shared_vgpr_count + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_shared_vgpr_count defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_shared_vgpr_count + +.set defined_boolean, 1 + +// ASM: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s new file mode 100644 index 000000000000..95af59c413ae --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s @@ -0,0 +1,190 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0afe4 801f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0afe4 801f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s new file mode 100644 index 000000000000..e1107fb69ba4 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s @@ -0,0 +1,186 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0afe4 811f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0afe4 811f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_enable_private_segment 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s new file mode 100644 index 000000000000..449616d35186 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s @@ -0,0 +1,184 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f02fe4 811f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f02fe4 811f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_round_robin_scheduling defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_round_robin_scheduling defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_enable_private_segment 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_round_robin_scheduling 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s new file mode 100644 index 000000000000..c7e05441b45f --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s @@ -0,0 +1,168 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s new file mode 100644 index 000000000000..49a5015987a6 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s @@ -0,0 +1,171 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefix=ASM %s + +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s new file mode 100644 index 000000000000..b7f89239160f --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s @@ -0,0 +1,148 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0030 0000ac04 81000000 00000000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0070 0000ac04 81000000 00000000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean + .amdhsa_dx10_clamp defined_boolean + .amdhsa_ieee_mode defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_tg_split defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean + .amdhsa_dx10_clamp defined_boolean + .amdhsa_ieee_mode defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_tg_split defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&63)>>0)+1)*4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&65536)>>16 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset 4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 0 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_tg_split 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 +// ASM-NEXT: .amdhsa_exception_int_div_zero 0 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-tg-split.s b/llvm/test/MC/AMDGPU/hsa-tg-split.s new file mode 100644 index 000000000000..5a4d3e2c279c --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-tg-split.s @@ -0,0 +1,74 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// OBJDUMP: Contents of section .rodata +// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000 + +.text +// ASM: .text + +.amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" +// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type minimal,@function +minimal: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel minimal + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel minimal +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset 4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 0 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 0 +// ASM-NEXT: .amdhsa_tg_split 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 +// ASM-NEXT: .amdhsa_exception_int_div_zero 0 +// ASM-NEXT: .end_amdhsa_kernel -- GitLab From 26c3d018b1f57b33779c53d4fed5ea895810c314 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 21 Mar 2024 13:57:14 +0000 Subject: [PATCH 050/933] [gn build] Port 857161c367a1 --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 12d875cf40c9..5ba91fcec83a 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -104,6 +104,7 @@ static_library("MCTargetDesc") { "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", "AMDGPUMCExpr.cpp", + "AMDGPUMCKernelDescriptor.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From a11d9b463966d31ecedb373115abdcca54f704c3 Mon Sep 17 00:00:00 2001 From: Akira Hatanaka Date: Thu, 21 Mar 2024 07:10:42 -0700 Subject: [PATCH 051/933] Disable driver tests on macosx that are currently disabled on darwin (#85990) macosx and darwin in triples are equivalent. rdar://124246653 --- clang/test/Driver/clang-offload-bundler-asserts-on.c | 2 +- clang/test/Driver/clang-offload-bundler-standardize.c | 2 +- clang/test/Driver/clang-offload-bundler.c | 2 +- clang/test/Driver/fat-archive-unbundle-ext.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/Driver/clang-offload-bundler-asserts-on.c b/clang/test/Driver/clang-offload-bundler-asserts-on.c index 521c8641ff54..eb11d5fbbee4 100644 --- a/clang/test/Driver/clang-offload-bundler-asserts-on.c +++ b/clang/test/Driver/clang-offload-bundler-asserts-on.c @@ -1,6 +1,6 @@ // REQUIRES: x86-registered-target // REQUIRES: asserts -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // Generate the file we can bundle. // RUN: %clang -O0 -target %itanium_abi_triple %s -c -o %t.o diff --git a/clang/test/Driver/clang-offload-bundler-standardize.c b/clang/test/Driver/clang-offload-bundler-standardize.c index 6a24968c30ef..91dc8947aabb 100644 --- a/clang/test/Driver/clang-offload-bundler-standardize.c +++ b/clang/test/Driver/clang-offload-bundler-standardize.c @@ -1,6 +1,6 @@ // REQUIRES: x86-registered-target // REQUIRES: asserts -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // REQUIRES: asserts // Generate the file we can bundle. diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c index f3cd2493e052..a56a5424abf8 100644 --- a/clang/test/Driver/clang-offload-bundler.c +++ b/clang/test/Driver/clang-offload-bundler.c @@ -1,5 +1,5 @@ // REQUIRES: x86-registered-target -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // // Generate all the types of files we can bundle. diff --git a/clang/test/Driver/fat-archive-unbundle-ext.c b/clang/test/Driver/fat-archive-unbundle-ext.c index b409aa6313b1..e98b872f0c0c 100644 --- a/clang/test/Driver/fat-archive-unbundle-ext.c +++ b/clang/test/Driver/fat-archive-unbundle-ext.c @@ -1,5 +1,5 @@ // REQUIRES: x86-registered-target -// UNSUPPORTED: target={{.*-windows.*}}, target={{.*-darwin.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*-windows.*}}, target={{.*}}-macosx{{.*}}, target={{.*-darwin.*}}, target={{.*}}-aix{{.*}} // Generate dummy fat object // RUN: %clang -O0 -target %itanium_abi_triple %s -c -o %t.host.o -- GitLab From 2152094a45af98c9ccfef6d5913f38c66ab8b165 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Thu, 21 Mar 2024 15:11:45 +0100 Subject: [PATCH 052/933] [clang] Improves -print-library-module-manifest-path. (#85943) This adds a libc++ to modules.json as is currently used by libc++. When libc++.so is not found the function will search for libc++.a as fallback. --- clang/lib/Driver/Driver.cpp | 49 +++++++++++-------- ...les-print-library-module-manifest-path.cpp | 19 ++++++- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 767c1cd47e8c..7a53764364ce 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -6203,28 +6203,35 @@ std::string Driver::GetStdModuleManifestPath(const Compilation &C, switch (TC.GetCXXStdlibType(C.getArgs())) { case ToolChain::CST_Libcxx: { - std::string lib = GetFilePath("libc++.so", TC); - - // Note when there are multiple flavours of libc++ the module json needs to - // look at the command-line arguments for the proper json. - // These flavours do not exist at the moment, but there are plans to - // provide a variant that is built with sanitizer instrumentation enabled. - - // For example - // StringRef modules = [&] { - // const SanitizerArgs &Sanitize = TC.getSanitizerArgs(C.getArgs()); - // if (Sanitize.needsAsanRt()) - // return "modules-asan.json"; - // return "modules.json"; - // }(); - - SmallString<128> path(lib.begin(), lib.end()); - llvm::sys::path::remove_filename(path); - llvm::sys::path::append(path, "modules.json"); - if (TC.getVFS().exists(path)) - return static_cast(path); + auto evaluate = [&](const char *library) -> std::optional { + std::string lib = GetFilePath(library, TC); + + // Note when there are multiple flavours of libc++ the module json needs + // to look at the command-line arguments for the proper json. These + // flavours do not exist at the moment, but there are plans to provide a + // variant that is built with sanitizer instrumentation enabled. + + // For example + // StringRef modules = [&] { + // const SanitizerArgs &Sanitize = TC.getSanitizerArgs(C.getArgs()); + // if (Sanitize.needsAsanRt()) + // return "libc++.modules-asan.json"; + // return "libc++.modules.json"; + // }(); + + SmallString<128> path(lib.begin(), lib.end()); + llvm::sys::path::remove_filename(path); + llvm::sys::path::append(path, "libc++.modules.json"); + if (TC.getVFS().exists(path)) + return static_cast(path); + + return {}; + }; - return error; + if (std::optional result = evaluate("libc++.so"); result) + return *result; + + return evaluate("libc++.a").value_or(error); } case ToolChain::CST_Libstdcxx: diff --git a/clang/test/Driver/modules-print-library-module-manifest-path.cpp b/clang/test/Driver/modules-print-library-module-manifest-path.cpp index 24797002b80f..3ba2709ad95c 100644 --- a/clang/test/Driver/modules-print-library-module-manifest-path.cpp +++ b/clang/test/Driver/modules-print-library-module-manifest-path.cpp @@ -3,6 +3,7 @@ // RUN: rm -rf %t && split-file %s %t && cd %t // RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu // RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libc++ \ @@ -10,13 +11,21 @@ // RUN: --target=x86_64-linux-gnu 2>&1 \ // RUN: | FileCheck libcxx-no-module-json.cpp -// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/modules.json +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libc++ \ // RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ // RUN: --target=x86_64-linux-gnu 2>&1 \ // RUN: | FileCheck libcxx.cpp +// RUN: rm %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a +// RUN: %clang -print-library-module-manifest-path \ +// RUN: -stdlib=libc++ \ +// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ +// RUN: --target=x86_64-linux-gnu 2>&1 \ +// RUN: | FileCheck libcxx-no-shared-lib.cpp + // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libstdc++ \ // RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ @@ -29,7 +38,13 @@ //--- libcxx.cpp -// CHECK: {{.*}}/Inputs/usr/lib/x86_64-linux-gnu{{/|\\}}modules.json +// CHECK: {{.*}}/Inputs/usr/lib/x86_64-linux-gnu{{/|\\}}libc++.modules.json + +//--- libcxx-no-shared-lib.cpp + +// Note this might find a different path depending whether search path +// contains a different libc++.so. +// CHECK: {{.*}}libc++.modules.json //--- libstdcxx.cpp -- GitLab From 2861856baf16e43a5e465e87022c6c2c2d238969 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Thu, 21 Mar 2024 14:18:56 +0000 Subject: [PATCH 053/933] [mlir][Vector] Add utility for computing scalable value bounds (#83876) This adds a new API built with the `ValueBoundsConstraintSet` to compute the bounds of possibly scalable quantities. It uses knowledge of the range of vscale (which is defined by the target architecture), to solve for the bound as either a constant or an expression in terms of vscale. The result is an `AffineMap` that will always take at most one parameter, vscale, and returns a single result, which is the bound of `value`. The API is defined as follows: ```c++ FailureOr vector::ScalableValueBoundsConstraintSet::computeScalableBound( Value value, std::optional dim, unsigned vscaleMin, unsigned vscaleMax, presburger::BoundType boundType, bool closedUB = true, StopConditionFn stopCondition = nullptr); ``` Note: `ConstantOrScalableBound` is a thin wrapper over the `AffineMap` with a utility for converting the bound to a single quantity (i.e. a size and scalable flag). We believe this API could prove useful downstream in IREE (which uses a similar analysis to hoist allocas, which currently fails for scalable vectors). --- .../IR/ScalableValueBoundsConstraintSet.h | 104 +++++++++++ .../Vector/IR/ValueBoundsOpInterfaceImpl.h | 20 +++ mlir/include/mlir/InitAllDialects.h | 2 + .../mlir/Interfaces/ValueBoundsOpInterface.h | 16 +- mlir/lib/Dialect/Vector/IR/CMakeLists.txt | 2 + .../IR/ScalableValueBoundsConstraintSet.cpp | 103 +++++++++++ .../Vector/IR/ValueBoundsOpInterfaceImpl.cpp | 51 ++++++ .../lib/Interfaces/ValueBoundsOpInterface.cpp | 86 +++++++--- .../Dialect/Vector/test-scalable-bounds.mlir | 161 ++++++++++++++++++ .../Dialect/Affine/TestReifyValueBounds.cpp | 38 ++++- 10 files changed, 555 insertions(+), 28 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h create mode 100644 mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp create mode 100644 mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp create mode 100644 mlir/test/Dialect/Vector/test-scalable-bounds.mlir diff --git a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h new file mode 100644 index 000000000000..31e19ff1ad39 --- /dev/null +++ b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h @@ -0,0 +1,104 @@ +//===- ScalableValueBoundsConstraintSet.h - Scalable Value Bounds ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H +#define MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H + +#include "mlir/Analysis/Presburger/IntegerRelation.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" +#include "mlir/Interfaces/ValueBoundsOpInterface.h" + +namespace mlir::vector { + +namespace detail { + +/// Parent class for the value bounds RTTIExtends. Uses protected inheritance to +/// hide all ValueBoundsConstraintSet methods by default (as some do not use the +/// ScalableValueBoundsConstraintSet, so may produce unexpected results). +struct ValueBoundsConstraintSet : protected ::mlir::ValueBoundsConstraintSet { + using ::mlir::ValueBoundsConstraintSet::ValueBoundsConstraintSet; +}; +} // namespace detail + +/// A version of `ValueBoundsConstraintSet` that can solve for scalable bounds. +struct ScalableValueBoundsConstraintSet + : public llvm::RTTIExtends { + ScalableValueBoundsConstraintSet(MLIRContext *context, unsigned vscaleMin, + unsigned vscaleMax) + : RTTIExtends(context), vscaleMin(vscaleMin), vscaleMax(vscaleMax){}; + + using RTTIExtends::bound; + using RTTIExtends::StopConditionFn; + + /// A thin wrapper over an `AffineMap` which can represent a constant bound, + /// or a scalable bound (in terms of vscale). The `AffineMap` will always + /// take at most one parameter, vscale, and returns a single result, which is + /// the bound of value. + struct ConstantOrScalableBound { + AffineMap map; + + struct BoundSize { + int64_t baseSize{0}; + bool scalable{false}; + }; + + /// Get the (possibly) scalable size of the bound, returns failure if + /// the bound cannot be represented as a single quantity. + FailureOr getSize() const; + }; + + /// Computes a (possibly) scalable bound for a given value. This is + /// similar to `ValueBoundsConstraintSet::computeConstantBound()`, but + /// uses knowledge of the range of vscale to compute either a constant + /// bound, an expression in terms of vscale, or failure if no bound can + /// be computed. + /// + /// The resulting `AffineMap` will always take at most one parameter, + /// vscale, and return a single result, which is the bound of `value`. + /// + /// Note: `vscaleMin` must be `<=` to `vscaleMax`. If `vscaleMin` == + /// `vscaleMax`, the resulting bound (if found), will be constant. + static FailureOr + computeScalableBound(Value value, std::optional dim, + unsigned vscaleMin, unsigned vscaleMax, + presburger::BoundType boundType, bool closedUB = true, + StopConditionFn stopCondition = nullptr); + + /// Get the value of vscale. Returns `nullptr` vscale as not been encountered. + Value getVscaleValue() const { return vscale; } + + /// Sets the value of vscale. Asserts if vscale has already been set. + void setVscale(vector::VectorScaleOp vscaleOp) { + assert(!vscale && "expected vscale to be unset"); + vscale = vscaleOp.getResult(); + } + + /// The minimum possible value of vscale. + unsigned getVscaleMin() const { return vscaleMin; } + + /// The maximum possible value of vscale. + unsigned getVscaleMax() const { return vscaleMax; } + + static char ID; + +private: + const unsigned vscaleMin; + const unsigned vscaleMax; + + // This will be set when the first `vector.vscale` operation is found within + // the `ValueBoundsOpInterface` implementation then reused from there on. + Value vscale = nullptr; +}; + +using ConstantOrScalableBound = + ScalableValueBoundsConstraintSet::ConstantOrScalableBound; + +} // namespace mlir::vector + +#endif // MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H diff --git a/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h new file mode 100644 index 000000000000..4794bc9016c6 --- /dev/null +++ b/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- ValueBoundsOpInterfaceImpl.h - Impl. of ValueBoundsOpInterface -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H +#define MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace vector { +void registerValueBoundsOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace vector +} // namespace mlir + +#endif // MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 21775e11e071..9bbf12d13254 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -82,6 +82,7 @@ #include "mlir/Dialect/Transform/IR/TransformDialect.h" #include "mlir/Dialect/Transform/PDLExtension/PDLExtension.h" #include "mlir/Dialect/UB/IR/UBOps.h" +#include "mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/Vector/Transforms/SubsetOpInterfaceImpl.h" @@ -174,6 +175,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { tosa::registerShardingInterfaceExternalModels(registry); vector::registerBufferizableOpInterfaceExternalModels(registry); vector::registerSubsetOpInterfaceExternalModels(registry); + vector::registerValueBoundsOpInterfaceExternalModels(registry); NVVM::registerNVVMTargetInterfaceExternalModels(registry); ROCDL::registerROCDLTargetInterfaceExternalModels(registry); spirv::registerSPIRVTargetInterfaceExternalModels(registry); diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h index 28dadfb9ecf8..b4ed0967e63f 100644 --- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h +++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h @@ -15,6 +15,7 @@ #include "mlir/IR/Value.h" #include "mlir/Interfaces/DestinationStyleOpInterface.h" #include "llvm/ADT/SetVector.h" +#include "llvm/Support/ExtensibleRTTI.h" #include @@ -63,7 +64,8 @@ using ValueDimList = SmallVector>>; /// /// Note: Any modification of existing IR invalides the data stored in this /// class. Adding new operations is allowed. -class ValueBoundsConstraintSet { +class ValueBoundsConstraintSet + : public llvm::RTTIExtends { protected: /// Helper class that builds a bound for a shaped value dimension or /// index-typed value. @@ -107,6 +109,8 @@ protected: }; public: + static char ID; + /// The stop condition when traversing the backward slice of a shaped value/ /// index-type value. The traversal continues until the stop condition /// evaluates to "true" for a value. @@ -265,6 +269,16 @@ protected: ValueBoundsConstraintSet(MLIRContext *ctx); + /// Populates the constraint set for a value/map without actually computing + /// the bound. Returns the position for the value/map (via the return value + /// and `posOut` output parameter). + int64_t populateConstraintsSet(Value value, + std::optional dim = std::nullopt, + StopConditionFn stopCondition = nullptr); + int64_t populateConstraintsSet(AffineMap map, ValueDimList mapOperands, + StopConditionFn stopCondition = nullptr, + int64_t *posOut = nullptr); + /// Iteratively process all elements on the worklist until an index-typed /// value or shaped value meets `stopCondition`. Such values are not processed /// any further. diff --git a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt index 70f3fa8c297d..204462ffd047 100644 --- a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt @@ -1,5 +1,7 @@ add_mlir_dialect_library(MLIRVectorDialect VectorOps.cpp + ValueBoundsOpInterfaceImpl.cpp + ScalableValueBoundsConstraintSet.cpp ADDITIONAL_HEADER_DIRS ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector/IR diff --git a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp new file mode 100644 index 000000000000..6d7e3bc70f59 --- /dev/null +++ b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp @@ -0,0 +1,103 @@ +//===- ScalableValueBoundsConstraintSet.cpp - Scalable Value Bounds -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" + +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +namespace mlir::vector { + +FailureOr +ConstantOrScalableBound::getSize() const { + if (map.isSingleConstant()) + return BoundSize{map.getSingleConstantResult(), /*scalable=*/false}; + if (map.getNumResults() != 1 || map.getNumInputs() != 1) + return failure(); + auto binop = dyn_cast(map.getResult(0)); + if (!binop || binop.getKind() != AffineExprKind::Mul) + return failure(); + auto matchConstant = [&](AffineExpr expr, int64_t &constant) -> bool { + if (auto cst = dyn_cast(expr)) { + constant = cst.getValue(); + return true; + } + return false; + }; + // Match `s0 * cst` or `cst * s0`: + int64_t cst = 0; + auto lhs = binop.getLHS(); + auto rhs = binop.getRHS(); + if ((matchConstant(lhs, cst) && isa(rhs)) || + (matchConstant(rhs, cst) && isa(lhs))) { + return BoundSize{cst, /*scalable=*/true}; + } + return failure(); +} + +char ScalableValueBoundsConstraintSet::ID = 0; + +FailureOr +ScalableValueBoundsConstraintSet::computeScalableBound( + Value value, std::optional dim, unsigned vscaleMin, + unsigned vscaleMax, presburger::BoundType boundType, bool closedUB, + StopConditionFn stopCondition) { + using namespace presburger; + + assert(vscaleMin <= vscaleMax); + ScalableValueBoundsConstraintSet scalableCstr(value.getContext(), vscaleMin, + vscaleMax); + + int64_t pos = scalableCstr.populateConstraintsSet(value, dim, stopCondition); + + // Project out all variables apart from vscale. + // This should result in constraints in terms of vscale only. + scalableCstr.projectOut( + [&](ValueDim p) { return p.first != scalableCstr.getVscaleValue(); }); + + assert(scalableCstr.cstr.getNumDimAndSymbolVars() == + scalableCstr.positionToValueDim.size() && + "inconsistent mapping state"); + + // Check that the only symbols left are vscale. + for (int64_t i = 0; i < scalableCstr.cstr.getNumDimAndSymbolVars(); ++i) { + if (i == pos) + continue; + if (scalableCstr.positionToValueDim[i] != + ValueDim(scalableCstr.getVscaleValue(), + ValueBoundsConstraintSet::kIndexValue)) { + return failure(); + } + } + + SmallVector lowerBound(1), upperBound(1); + scalableCstr.cstr.getSliceBounds(pos, 1, value.getContext(), &lowerBound, + &upperBound, closedUB); + + auto invalidBound = [](auto &bound) { + return !bound[0] || bound[0].getNumResults() != 1; + }; + + AffineMap bound = [&] { + if (boundType == BoundType::EQ && !invalidBound(lowerBound) && + lowerBound[0] == lowerBound[0]) { + return lowerBound[0]; + } else if (boundType == BoundType::LB && !invalidBound(lowerBound)) { + return lowerBound[0]; + } else if (boundType == BoundType::UB && !invalidBound(upperBound)) { + return upperBound[0]; + } + return AffineMap{}; + }(); + + if (!bound) + return failure(); + + return ConstantOrScalableBound{bound}; +} + +} // namespace mlir::vector diff --git a/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp new file mode 100644 index 000000000000..ca95072d9bb0 --- /dev/null +++ b/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp @@ -0,0 +1,51 @@ +//===- ValueBoundsOpInterfaceImpl.cpp - Impl. of ValueBoundsOpInterface ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h" + +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" +#include "mlir/Interfaces/ValueBoundsOpInterface.h" + +using namespace mlir; + +namespace mlir::vector { +namespace { + +struct VectorScaleOpInterface + : public ValueBoundsOpInterface::ExternalModel { + void populateBoundsForIndexValue(Operation *op, Value value, + ValueBoundsConstraintSet &cstr) const { + auto *scalableCstr = dyn_cast(&cstr); + if (!scalableCstr) + return; + auto vscaleOp = cast(op); + assert(value == vscaleOp.getResult() && "invalid value"); + if (auto vscale = scalableCstr->getVscaleValue()) { + // All copies of vscale are equivalent. + scalableCstr->bound(value) == cstr.getExpr(vscale); + } else { + // We know vscale is confined to [vscaleMin, vscaleMax]. + scalableCstr->bound(value) >= scalableCstr->getVscaleMin(); + scalableCstr->bound(value) <= scalableCstr->getVscaleMax(); + scalableCstr->setVscale(vscaleOp); + } + } +}; + +} // namespace +} // namespace mlir::vector + +void mlir::vector::registerValueBoundsOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, vector::VectorDialect *dialect) { + vector::VectorScaleOp::attachInterface( + *ctx); + }); +} diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp index 85abc2df8947..06ec3f4e135e 100644 --- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp +++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp @@ -70,6 +70,8 @@ static std::optional getConstantIntValue(OpFoldResult ofr) { ValueBoundsConstraintSet::ValueBoundsConstraintSet(MLIRContext *ctx) : builder(ctx) {} +char ValueBoundsConstraintSet::ID = 0; + #ifndef NDEBUG static void assertValidValueDim(Value value, std::optional dim) { if (value.getType().isIndex()) { @@ -471,55 +473,87 @@ FailureOr ValueBoundsConstraintSet::computeConstantBound( closedUB); } +FailureOr ValueBoundsConstraintSet::computeConstantBound( + presburger::BoundType type, AffineMap map, ArrayRef operands, + StopConditionFn stopCondition, bool closedUB) { + ValueDimList valueDims; + for (Value v : operands) { + assert(v.getType().isIndex() && "expected index type"); + valueDims.emplace_back(v, std::nullopt); + } + return computeConstantBound(type, map, valueDims, stopCondition, closedUB); +} + FailureOr ValueBoundsConstraintSet::computeConstantBound( presburger::BoundType type, AffineMap map, ValueDimList operands, StopConditionFn stopCondition, bool closedUB) { assert(map.getNumResults() == 1 && "expected affine map with one result"); ValueBoundsConstraintSet cstr(map.getContext()); - int64_t pos = cstr.insert(/*isSymbol=*/false); + + int64_t pos = 0; + if (stopCondition) { + cstr.populateConstraintsSet(map, operands, stopCondition, &pos); + } else { + // No stop condition specified: Keep adding constraints until a bound could + // be computed. + cstr.populateConstraintsSet( + map, operands, + [&](Value v, std::optional dim) { + return cstr.cstr.getConstantBound64(type, pos).has_value(); + }, + &pos); + } + // Compute constant bound for `valueDim`. + int64_t ubAdjustment = closedUB ? 0 : 1; + if (auto bound = cstr.cstr.getConstantBound64(type, pos)) + return type == BoundType::UB ? *bound + ubAdjustment : *bound; + return failure(); +} + +int64_t ValueBoundsConstraintSet::populateConstraintsSet( + Value value, std::optional dim, StopConditionFn stopCondition) { +#ifndef NDEBUG + assertValidValueDim(value, dim); +#endif // NDEBUG + + AffineMap map = + AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, + Builder(value.getContext()).getAffineDimExpr(0)); + return populateConstraintsSet(map, {{value, dim}}, stopCondition); +} + +int64_t ValueBoundsConstraintSet::populateConstraintsSet( + AffineMap map, ValueDimList operands, StopConditionFn stopCondition, + int64_t *posOut) { + assert(map.getNumResults() == 1 && "expected affine map with one result"); + int64_t pos = insert(/*isSymbol=*/false); + if (posOut) + *posOut = pos; // Add map and operands to the constraint set. Dimensions are converted to // symbols. All operands are added to the worklist. auto mapper = [&](std::pair> v) { - return cstr.getExpr(v.first, v.second); + return getExpr(v.first, v.second); }; SmallVector dimReplacements = llvm::to_vector( llvm::map_range(ArrayRef(operands).take_front(map.getNumDims()), mapper)); SmallVector symReplacements = llvm::to_vector( llvm::map_range(ArrayRef(operands).drop_front(map.getNumDims()), mapper)); - cstr.addBound( + addBound( presburger::BoundType::EQ, pos, map.getResult(0).replaceDimsAndSymbols(dimReplacements, symReplacements)); // Process the backward slice of `operands` (i.e., reverse use-def chain) // until `stopCondition` is met. if (stopCondition) { - cstr.processWorklist(stopCondition); + processWorklist(stopCondition); } else { - // No stop condition specified: Keep adding constraints until a bound could - // be computed. - cstr.processWorklist( - /*stopCondition=*/[&](Value v, std::optional dim) { - return cstr.cstr.getConstantBound64(type, pos).has_value(); - }); + // No stop condition specified: Keep adding constraints until the worklist + // is empty. + processWorklist([](Value v, std::optional dim) { return false; }); } - // Compute constant bound for `valueDim`. - int64_t ubAdjustment = closedUB ? 0 : 1; - if (auto bound = cstr.cstr.getConstantBound64(type, pos)) - return type == BoundType::UB ? *bound + ubAdjustment : *bound; - return failure(); -} - -FailureOr ValueBoundsConstraintSet::computeConstantBound( - presburger::BoundType type, AffineMap map, ArrayRef operands, - StopConditionFn stopCondition, bool closedUB) { - ValueDimList valueDims; - for (Value v : operands) { - assert(v.getType().isIndex() && "expected index type"); - valueDims.emplace_back(v, std::nullopt); - } - return computeConstantBound(type, map, valueDims, stopCondition, closedUB); + return pos; } FailureOr diff --git a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir new file mode 100644 index 000000000000..245a6f5c13ac --- /dev/null +++ b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir @@ -0,0 +1,161 @@ +// RUN: mlir-opt %s -test-affine-reify-value-bounds -cse -verify-diagnostics \ +// RUN: -verify-diagnostics -split-input-file | FileCheck %s + +#map_dim_i = affine_map<(d0)[s0] -> (-d0 + 32400, s0)> +#map_dim_j = affine_map<(d0)[s0] -> (-d0 + 16, s0)> + +// Here the upper bound for min_i is 4 x vscale, as we know 4 x vscale is +// always less than 32400. The bound for min_j is 16, as 16 is always less +// 4 x vscale_max (vscale_max is the UB for vscale). + +// CHECK: #[[$SCALABLE_BOUND_MAP_0:.*]] = affine_map<()[s0] -> (s0 * 4)> + +// CHECK-LABEL: @fixed_size_loop_nest +// CHECK-DAG: %[[VSCALE:.*]] = vector.vscale +// CHECK-DAG: %[[UB_i:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_0]]()[%[[VSCALE]]] +// CHECK-DAG: %[[UB_j:.*]] = arith.constant 16 : index +// CHECK: "test.some_use"(%[[UB_i]], %[[UB_j]]) : (index, index) -> () +func.func @fixed_size_loop_nest() { + %c16 = arith.constant 16 : index + %c32400 = arith.constant 32400 : index + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + %vscale = vector.vscale + %c4_vscale = arith.muli %vscale, %c4 : index + scf.for %i = %c0 to %c32400 step %c4_vscale { + %min_i = affine.min #map_dim_i(%i)[%c4_vscale] + scf.for %j = %c0 to %c16 step %c4_vscale { + %min_j = affine.min #map_dim_j(%j)[%c4_vscale] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i, %bound_j) : (index, index) -> () + } + } + return +} + +// ----- + +#map_dynamic_dim = affine_map<(d0)[s0, s1] -> (-d0 + s1, s0)> + +// Here upper bounds for both min_i and min_j are both (conservatively) +// 4 x vscale, as we know that is always the largest value they could take. As +// if `dim < 4 x vscale` then 4 x vscale is an overestimate, and if +// `dim > 4 x vscale` then the min will be clamped to 4 x vscale. + +// CHECK: #[[$SCALABLE_BOUND_MAP_1:.*]] = affine_map<()[s0] -> (s0 * 4)> + +// CHECK-LABEL: @dynamic_size_loop_nest +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[UB_ij:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_1]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[UB_ij]], %[[UB_ij]]) : (index, index) -> () +func.func @dynamic_size_loop_nest(%dim0: index, %dim1: index) { + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + %vscale = vector.vscale + %c4_vscale = arith.muli %vscale, %c4 : index + scf.for %i = %c0 to %dim0 step %c4_vscale { + %min_i = affine.min #map_dynamic_dim(%i)[%c4_vscale, %dim0] + scf.for %j = %c0 to %dim1 step %c4_vscale { + %min_j = affine.min #map_dynamic_dim(%j)[%c4_vscale, %dim1] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i, %bound_j) : (index, index) -> () + } + } + return +} + +// ----- + +// Here the bound is just a value + a constant. + +// CHECK: #[[$SCALABLE_BOUND_MAP_2:.*]] = affine_map<()[s0] -> (s0 + 8)> + +// CHECK-LABEL: @add_to_vscale +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[SCALABLE_BOUND:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_2]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[SCALABLE_BOUND]]) : (index) -> () +func.func @add_to_vscale() { + %vscale = vector.vscale + %c8 = arith.constant 8 : index + %vscale_plus_c8 = arith.addi %vscale, %c8 : index + %bound = "test.reify_scalable_bound"(%vscale_plus_c8) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we know vscale is always 2 so we get a constant bound. + +// CHECK-LABEL: @vscale_fixed_size +// CHECK: %[[C2:.*]] = arith.constant 2 : index +// CHECK: "test.some_use"(%[[C2]]) : (index) -> () +func.func @vscale_fixed_size() { + %vscale = vector.vscale + %bound = "test.reify_scalable_bound"(%vscale) {type = "EQ", vscale_min = 2, vscale_max = 2} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we don't know the upper bound (%a is underspecified) + +func.func @unknown_bound(%a: index) { + %vscale = vector.vscale + %vscale_plus_a = arith.muli %vscale, %a : index + // expected-error @below{{could not reify bound}} + %bound = "test.reify_scalable_bound"(%vscale_plus_a) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we have two vscale values (that have not been CSE'd), but they should +// still be treated as equivalent. + +// CHECK: #[[$SCALABLE_BOUND_MAP_3:.*]] = affine_map<()[s0] -> (s0 * 6)> + +// CHECK-LABEL: @duplicate_vscale_values +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[SCALABLE_BOUND:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_3]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[SCALABLE_BOUND]]) : (index) -> () +func.func @duplicate_vscale_values() { + %c4 = arith.constant 4 : index + %vscale_0 = vector.vscale + + %c2 = arith.constant 2 : index + %vscale_1 = vector.vscale + + %c4_vscale = arith.muli %vscale_0, %c4 : index + %c2_vscale = arith.muli %vscale_1, %c2 : index + %add = arith.addi %c2_vscale, %c4_vscale : index + + %bound = "test.reify_scalable_bound"(%add) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Test some non-scalable code to ensure that works too: + +#map_dim_i = affine_map<(d0)[s0] -> (-d0 + 1024, s0)> + +// CHECK-LABEL: @non_scalable_code +// CHECK: %[[C4:.*]] = arith.constant 4 : index +// CHECK: "test.some_use"(%[[C4]]) : (index) -> () +func.func @non_scalable_code() { + %c1024 = arith.constant 1024 : index + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + scf.for %i = %c0 to %c1024 step %c4 { + %min_i = affine.min #map_dim_i(%i)[%c4] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i) : (index) -> () + } + return +} diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp index 39671a930f2e..5e160b720db6 100644 --- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp +++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp @@ -13,6 +13,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" #include "mlir/IR/PatternMatch.h" #include "mlir/Interfaces/ValueBoundsOpInterface.h" #include "mlir/Pass/Pass.h" @@ -75,7 +76,8 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, WalkResult result = funcOp.walk([&](Operation *op) { // Look for test.reify_bound ops. if (op->getName().getStringRef() == "test.reify_bound" || - op->getName().getStringRef() == "test.reify_constant_bound") { + op->getName().getStringRef() == "test.reify_constant_bound" || + op->getName().getStringRef() == "test.reify_scalable_bound") { if (op->getNumOperands() != 1 || op->getNumResults() != 1 || !op->getResultTypes()[0].isIndex()) { op->emitOpError("invalid op"); @@ -110,6 +112,9 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, bool constant = op->getName().getStringRef() == "test.reify_constant_bound"; + bool scalable = !constant && op->getName().getStringRef() == + "test.reify_scalable_bound"; + // Prepare stop condition. By default, reify in terms of the op's // operands. No stop condition is used when a constant was requested. std::function)> stopCondition = @@ -137,6 +142,37 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, if (succeeded(reifiedConst)) reified = FailureOr(rewriter.getIndexAttr(*reifiedConst)); + } else if (scalable) { + unsigned vscaleMin = 0; + unsigned vscaleMax = 0; + if (auto attr = "vscale_min"; op->hasAttrOfType(attr)) { + vscaleMin = unsigned(op->getAttrOfType(attr).getInt()); + } else { + op->emitOpError("expected `vscale_min` to be provided"); + return WalkResult::skip(); + } + if (auto attr = "vscale_max"; op->hasAttrOfType(attr)) { + vscaleMax = unsigned(op->getAttrOfType(attr).getInt()); + } else { + op->emitOpError("expected `vscale_max` to be provided"); + return WalkResult::skip(); + } + + auto loc = op->getLoc(); + auto reifiedScalable = + vector::ScalableValueBoundsConstraintSet::computeScalableBound( + value, dim, vscaleMin, vscaleMax, *boundType); + if (succeeded(reifiedScalable)) { + SmallVector>, 1> + vscaleOperand; + if (reifiedScalable->map.getNumInputs() == 1) { + // The only possible input to the bound is vscale. + vscaleOperand.push_back(std::make_pair( + rewriter.create(loc), std::nullopt)); + } + reified = affine::materializeComputedBound( + rewriter, loc, reifiedScalable->map, vscaleOperand); + } } else { if (dim) { if (useArithOps) { -- GitLab From d1f182c895728d89c5c3d198b133e212a5d9d4a3 Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Thu, 21 Mar 2024 10:42:39 -0400 Subject: [PATCH 054/933] [AMDGPU][MC][True16] Rename and combine VINTERP MC tests (#85949) NFC. gfx11_asm_vinterp.s already contained GFX12 run lines. Rename the assembler and disassembler tests to be sorted based on real16 or fake16 instead of gfxip. Note, both GFX11 and GFX12 currently only have fake16 (fake16 in encoding, but not by name) upstream, so that is why the test files have a -fake16 suffix. One test input is changed, and that is the disassembler test for unsupported bits in the instruction. It is now an input that is valid on both GFX11 and GFX12. This was necessary because the size of the opcode field changed. --- llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s | 278 ------------------ llvm/test/MC/AMDGPU/vinterp-fake16.s | 182 ++++++++++++ .../AMDGPU/gfx11_dasm_vinterp.txt | 251 ---------------- .../AMDGPU/gfx12_dasm_vinterp.txt | 251 ---------------- .../MC/Disassembler/AMDGPU/vinterp-fake16.txt | 252 ++++++++++++++++ 5 files changed, 434 insertions(+), 780 deletions(-) delete mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s create mode 100644 llvm/test/MC/AMDGPU/vinterp-fake16.s delete mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt delete mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt create mode 100644 llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s deleted file mode 100644 index fdfbf65c0e3c..000000000000 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s +++ /dev/null @@ -1,278 +0,0 @@ -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GCN %s -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GCN %s - -v_interp_p10_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v1, v10, v20, v30 -// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] - -v_interp_p10_f32 v2, v11, v21, v31 -// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] - -v_interp_p10_f32 v3, v12, v22, v32 -// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v1, v10, v20, v30 -// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] - -v_interp_p2_f32 v2, v11, v21, v31 -// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] - -v_interp_p2_f32 v3, v12, v22, v32 -// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p2_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] diff --git a/llvm/test/MC/AMDGPU/vinterp-fake16.s b/llvm/test/MC/AMDGPU/vinterp-fake16.s new file mode 100644 index 000000000000..33dacdd92c31 --- /dev/null +++ b/llvm/test/MC/AMDGPU/vinterp-fake16.s @@ -0,0 +1,182 @@ +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -show-encoding %s | FileCheck -check-prefix=GCN %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s | FileCheck -check-prefix=GCN %s + +v_interp_p10_f32 v0, v1, v2, v3 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v1, v10, v20, v30 +// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] + +v_interp_p10_f32 v2, v11, v21, v31 +// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] + +v_interp_p10_f32 v3, v12, v22, v32 +// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 clamp +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, -v1, v2, v3 +// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_f32 v0, v1, -v2, v3 +// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_f32 v0, v1, v2, -v3 +// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v1, v10, v20, v30 +// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] + +v_interp_p2_f32 v2, v11, v21, v31 +// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] + +v_interp_p2_f32 v3, v12, v22, v32 +// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 clamp +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, -v1, v2, v3 +// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_f32 v0, v1, -v2, v3 +// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_f32 v0, v1, v2, -v3 +// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt deleted file mode 100644 index b22fd5e289fa..000000000000 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt +++ /dev/null @@ -1,251 +0,0 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX11 %s - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# Check that unused bits in the encoding are ignored. -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x80,0xcd,0x01,0x05,0x0e,0x1c - -# GFX11: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX11: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX11: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX11: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX11: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt deleted file mode 100644 index 977cd732947c..000000000000 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt +++ /dev/null @@ -1,251 +0,0 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# Check that unused bits in the encoding are ignored. -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c - -# GFX12: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX12: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX12: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX12: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX12: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt b/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt new file mode 100644 index 000000000000..239f1d8b3058 --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt @@ -0,0 +1,252 @@ +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble %s | FileCheck -strict-whitespace -check-prefix=CHECK %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble %s | FileCheck -strict-whitespace -check-prefix=CHECK %s + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# Check that unused bits in the encoding are ignored. +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x80,0xcd,0x01,0x05,0x0e,0x1c + +# CHECK: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 + +# CHECK: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 + +# CHECK: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 + +# CHECK: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 + +# CHECK: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 -- GitLab From 44278f2326e0df463f6d5dd43820858ee4cffc6f Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Thu, 21 Mar 2024 10:43:37 -0400 Subject: [PATCH 055/933] [AMDGPU][MC] Fix GFX12 check line typo and move test NFC. Fix CHECK lines that seem to have a copy paste error. Move the test that was formerly in gfx12_dasm_vinterp.txt (see #85949). --- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt index 0c4427cff63e..1be97b242284 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt @@ -12,8 +12,13 @@ # GFX12: v_add3_u32_e64_dpp v5, v1, 42, v0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x54,0x01,0x04,0x01,0x77,0x39,0x05] 0x05,0x00,0x55,0xd6,0xe9,0x54,0x01,0x04,0x01,0x77,0x39,0x05 -# GFX1150: v_add3_u32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05] +# GFX12: v_add3_u32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05] 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05 -# GFX1150: v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05] +# GFX12: v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05] 0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05 + +# Check that unused bits in the encoding are ignored. +# This is more strict than the check in vinterp-fake16.txt and is GFX12 specific. +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c -- GitLab From b4b5e8277a86d441830dbba54917bf22b0ad8608 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Thu, 21 Mar 2024 11:00:08 -0400 Subject: [PATCH 056/933] Check for all frame instructions in finalize isel. (#85945) Check for all frame instructions in finalize isel, not just for the frame setup opcode. This was proven necessary, see #78001 for discussion. --- llvm/lib/CodeGen/FinalizeISel.cpp | 3 +-- llvm/test/CodeGen/SystemZ/frame-adjstack.ll | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/SystemZ/frame-adjstack.ll diff --git a/llvm/lib/CodeGen/FinalizeISel.cpp b/llvm/lib/CodeGen/FinalizeISel.cpp index 978355f8eb1b..bf967eac22f1 100644 --- a/llvm/lib/CodeGen/FinalizeISel.cpp +++ b/llvm/lib/CodeGen/FinalizeISel.cpp @@ -59,8 +59,7 @@ bool FinalizeISel::runOnMachineFunction(MachineFunction &MF) { // Set AdjustsStack to true if the instruction selector emits a stack // frame setup instruction or a stack aligning inlineasm. - if (MI.getOpcode() == TII->getCallFrameSetupOpcode() || - MI.isStackAligningInlineAsm()) + if (TII->isFrameInstr(MI) || MI.isStackAligningInlineAsm()) MF.getFrameInfo().setAdjustsStack(true); // If MI is a pseudo, expand it. diff --git a/llvm/test/CodeGen/SystemZ/frame-adjstack.ll b/llvm/test/CodeGen/SystemZ/frame-adjstack.ll new file mode 100644 index 000000000000..7edacaa3d7d7 --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/frame-adjstack.ll @@ -0,0 +1,16 @@ +; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs | FileCheck %s +; +; Test that inserting a new MBB near a call during finalize isel custom +; insertion does not cause all frame instructions to be missed. That would +; result in a missing to set the AdjustsStack flag. + +; CHECK-LABEL: fun +define void @fun(i1 %cc) { + %sel = select i1 %cc, i32 5, i32 0 + tail call void @input_report_abs(i32 %sel) + %sel2 = select i1 %cc, i32 6, i32 1 + tail call void @input_report_abs(i32 %sel2) + ret void +} + +declare void @input_report_abs(i32) -- GitLab From 6898147b9f38c3bb46be5704bdad22abed7af339 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 21 Mar 2024 16:00:53 +0100 Subject: [PATCH 057/933] [InstCombine] Add contributor guide (#79007) Document expectations for contributions to InstCombine, especially regarding test coverage and alive2 proofs. --- llvm/docs/InstCombineContributorGuide.md | 556 +++++++++++++++++++++++ llvm/docs/UserGuides.rst | 5 + 2 files changed, 561 insertions(+) create mode 100644 llvm/docs/InstCombineContributorGuide.md diff --git a/llvm/docs/InstCombineContributorGuide.md b/llvm/docs/InstCombineContributorGuide.md new file mode 100644 index 000000000000..2416fd0920f6 --- /dev/null +++ b/llvm/docs/InstCombineContributorGuide.md @@ -0,0 +1,556 @@ +# InstCombine contributor guide + +This guide lays out a series of rules that contributions to InstCombine should +follow. **Following these rules will results in much faster PR approvals.** + +## Tests + +### Precommit tests + +Tests for new optimizations or miscompilation fixes should be pre-committed. +This means that you first commit the test with CHECK lines showing the behavior +*without* your change. Your actual change will then only contain CHECK line +diffs relative to that baseline. + +This means that pull requests should generally contain two commits: First, +one commit adding new tests with baseline check lines. Second, a commit with +functional changes and test diffs. + +If the second commit in your PR does not contain test diffs, you did something +wrong. Either you made a mistake when generating CHECK lines, or your tests are +not actually affected by your patch. + +Exceptions: When fixing assertion failures or infinite loops, do not pre-commit +tests. + +### Use `update_test_checks.py` + +CHECK lines should be generated using the `update_test_checks.py` script. Do +**not** manually edit check lines after using it. + +Be sure to use the correct opt binary when using the script. For example, if +your build directory is `build`, then you'll want to run: + +```sh +llvm/utils/update_test_checks.py --opt-binary build/bin/opt \ + llvm/test/Transforms/InstCombine/the_test.ll +``` + +Exceptions: Hand-written CHECK lines are allowed for debuginfo tests. + +### General testing considerations + +Place all tests relating to a transform into a single file. If you are adding +a regression test for a crash/miscompile in an existing transform, find the +file where the existing tests are located. A good way to do that is to comment +out the transform and see which tests fail. + +Make tests minimal. Only test exactly the pattern being transformed. If your +original motivating case is a larger pattern that your fold enables to +optimize in some non-trivial way, you may add it as well -- however, the bulk +of the test coverage should be minimal. + +Give tests short, but meaningful names. Don't call them `@test1`, `@test2` etc. +For example, a test checking multi-use behavior of a fold involving the +addition of two selects might be called `@add_of_selects_multi_use`. + +Add representative tests for each test category (discussed below), but don't +test all combinations of everything. If you have multi-use tests, and you have +commuted tests, you shouldn't also add commuted multi-use tests. + +Prefer to keep bit-widths for tests low to improve performance of proof checking using alive2. Using `i8` is better than `i128` where possible. + +### Add negative tests + +Make sure to add tests for which your transform does **not** apply. Start with +one of the test cases that succeeds and then create a sequence of negative +tests, such that **exactly one** different pre-condition of your transform is +not satisfied in each test. + +### Add multi-use tests + +Add multi-use tests that ensures your transform does not increase instruction +count if some instructions have additional uses. The standard pattern is to +introduce extra uses with function calls: + +```llvm +declare void @use(i8) + +define i8 @add_mul_const_multi_use(i8 %x) { + %add = add i8 %x, 1 + call void @use(i8 %add) + %mul = mul i8 %add, 3 + ret i8 %mul +} +``` + +Exceptions: For transform that only produce one instruction, multi-use tests +may be omitted. + +### Add commuted tests + +If the transform involves commutative operations, add tests with commuted +(swapped) operands. + +Make sure that the operand order stays intact in the CHECK lines of your +pre-commited tests. You should not see something like this: + +```llvm +; CHECK-NEXT: [[OR:%.*]] = or i8 [[X]], [[Y]] +; ... +%or = or i8 %y, %x +``` + +If this happens, you may need to change one of the operands to have higher +complexity (include the "thwart" comment in that case): + +```llvm +%y2 = mul i8 %y, %y ; thwart complexity-based canonicalization +%or = or i8 %y, %x +``` + +### Add vector tests + +When possible, it is recommended to add at least one test that uses vectors +instead of scalars. + +For patterns that include constants, we distinguish three kinds of tests. +The first are "splat" vectors, where all the vector elements are the same. +These tests *should* usually fold without additional effort. + +```llvm +define <2 x i8> @add_mul_const_vec_splat(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +A minor variant is to replace some of the splat elements with poison. These +will often also fold without additional effort. + +```llvm +define <2 x i8> @add_mul_const_vec_splat_poison(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +Finally, you can have non-splat vectors, where the vector elements are not +the same: + +```llvm +define <2 x i8> @add_mul_const_vec_non_splat(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +Non-splat vectors will often not fold by default. You should **not** try to +make them fold, unless doing so does not add **any** additional complexity. +You should still add the test though, even if it does not fold. + +### Flag tests + +If your transform involves instructions that can have poison-generating flags, +such as `nuw` and `nsw` on `add`, you should test how these interact with the +transform. + +If your transform *requires* a certain flag for correctness, make sure to add +negative tests missing the required flag. + +If your transform doesn't require flags for correctness, you should have tests +for preservation behavior. If the input instructions have certain flags, are +they preserved in the output instructions, if it is valid to preserve them? +(This depends on the transform. Check with alive2.) + +The same also applies to fast-math-flags (FMF). In that case, please always +test specific flags like `nnan`, `nsz` or `reassoc`, rather than the umbrella +`fast` flag. + +### Other tests + +The test categories mentioned above are non-exhaustive. There may be more tests +to be added, depending on the instructions involved in the transform. Some +examples: + + * For folds involving memory accesses like load/store, check that scalable vectors and non-byte-size types (like i3) are handled correctly. Also check that volatile/atomic are handled. + * For folds that interact with the bitwidth in some non-trivial way, check an illegal type like i13. Also confirm that the transform is correct for i1. + * For folds that involve phis, you may want to check that the case of multiple incoming values from one block is handled correctly. + +## Proofs + +Your pull request description should contain one or more +[alive2 proofs](https://alive2.llvm.org/ce/) for the correctness of the +proposed transform. + +### Basics + +Proofs are written using LLVM IR, by specifying a `@src` and `@tgt` function. +It is possible to include multiple proofs in a single file by giving the src +and tgt functions matching suffixes. + +For example, here is a pair of proofs that both `(x-y)+y` and `(x+y)-y` can +be simplified to `x` ([online](https://alive2.llvm.org/ce/z/MsPPGz)): + +```llvm +define i8 @src_add_sub(i8 %x, i8 %y) { + %add = add i8 %x, %y + %sub = sub i8 %add, %y + ret i8 %sub +} + +define i8 @tgt_add_sub(i8 %x, i8 %y) { + ret i8 %x +} + + +define i8 @src_sub_add(i8 %x, i8 %y) { + %sub = sub i8 %x, %y + %add = add i8 %sub, %y + ret i8 %add +} + +define i8 @tgt_sub_add(i8 %x, i8 %y) { + ret i8 %x +} +``` + +### Use generic values in proofs + +Proofs should operate on generic values, rather than specific constants, to the degree that this is possible. + +For example, if we want to fold `X s/ C s< X` to `X s> 0`, the following would +be a *bad* proof: + +```llvm +; Don't do this! +define i1 @src(i8 %x) { + %div = sdiv i8 %x, 123 + %cmp = icmp slt i8 %div, %x + ret i1 %cmp +} + +define i1 @tgt(i8 %x) { + %cmp = icmp sgt i8 %x, 0 + ret i1 %cmp +} +``` + +This is because it only proves that the transform is correct for the specific +constant 123. Maybe there are some constants for which the transform is +incorrect? + +The correct way to write this proof is as follows +([online](https://alive2.llvm.org/ce/z/acjwb6)): + +```llvm +define i1 @src(i8 %x, i8 %C) { + %precond = icmp ne i8 %C, 1 + call void @llvm.assume(i1 %precond) + %div = sdiv i8 %x, %C + %cmp = icmp slt i8 %div, %x + ret i1 %cmp +} + +define i1 @tgt(i8 %x, i8 %C) { + %cmp = icmp sgt i8 %x, 0 + ret i1 %cmp +} +``` + +Note that the `@llvm.assume` intrinsic is used to specify pre-conditions for +the transform. In this case, the proof will fail unless we specify `C != 1` as +a pre-condition. + +It should be emphasized that there is, in general, no expectation that the +IR in the proofs will be transformed by the implemented fold. In the above +example, the transform would only apply if `%C` is actually a constant, but we +need to use non-constants in the proof. + +### Common pre-conditions + +Here are some examples of common preconditions. + +```llvm +; %x is non-negative: +%nonneg = icmp sgt i8 %x, -1 +call void @llvm.assume(i1 %nonneg) + +; %x is a power of two: +%ctpop = call i8 @llvm.ctpop.i8(i8 %x) +%pow2 = icmp eq i8 %x, 1 +call void @llvm.assume(i1 %pow2) + +; %x is a power of two or zero: +%ctpop = call i8 @llvm.ctpop.i8(i8 %x) +%pow2orzero = icmp ult i8 %x, 2 +call void @llvm.assume(i1 %pow2orzero) + +; Adding %x and %y does not overflow in a signed sense: +%wo = call { i8, i1 } @llvm.sadd.with.overflow(i8 %x, i8 %y) +%ov = extractvalue { i8, i1 } %wo, 1 +%ov.not = xor i1 %ov, true +call void @llvm.assume(i1 %ov.not) +``` + +### Timeouts + +Alive2 proofs will sometimes produce a timeout with the following message: + +``` +Alive2 timed out while processing your query. +There are a few things you can try: + +- remove extraneous instructions, if any + +- reduce variable widths, for example to i16, i8, or i4 + +- add the --disable-undef-input command line flag, which + allows Alive2 to assume that arguments to your IR are not + undef. This is, in general, unsound: it can cause Alive2 + to miss bugs. +``` + +This is good advice, follow it! + +Reducing the bitwidth usually helps. For floating point numbers, you can use +the `half` type for bitwidth reduction purposes. For pointers, you can reduce +the bitwidth by specifying a custom data layout: + +```llvm +; For 16-bit pointers +target datalayout = "p:16:16" +``` + +If reducing the bitwidth does not help, try `-disable-undef-input`. This will +often significantly improve performance, but also implies that the correctness +of the transform with `undef` values is no longer verified. This is usually +fine if the transform does not increase the number of uses of any value. + +Finally, it's possible to build alive2 locally and use `-smt-to=` to verify +the proof with a larger timeout. If you don't want to do this (or it still +does not work), please submit the proof you have despite the timeout. + +## Implementation + +### Real-world usefulness + +There is a very large number of transforms that *could* be implemented, but +only a tiny fraction of them are useful for real-world code. + +Transforms that do not have real-world usefulness provide *negative* value to +the LLVM project, by taking up valuable reviewer time, increasing code +complexity and increasing compile-time overhead. + +We do not require explicit proof of real-world usefulness for every transform +-- in most cases the usefulness is fairly "obvious". However, the question may +come up for complex or unusual folds. Keep this in mind when chosing what you +work on. + +In particular, fixes for fuzzer-generated missed optimization reports will +likely be rejected if there is no evidence of real-world usefulness. + +### Pick the correct optimization pass + +There are a number of passes and utilities in the InstCombine family, and it +is important to pick the right place when implementing a fold. + + * `ConstantFolding`: For folding instructions with constant arguments to a constant. (Mainly relevant for intrinsics.) + * `ValueTracking`: For analyzing instructions, e.g. for known bits, non-zero, etc. Tests should usually use `-passes=instsimplify`. + * `InstructionSimplify`: For folds that do not create new instructions (either fold to existing value or constant). + * `InstCombine`: For folds that create or modify instructions. + * `AggressiveInstCombine`: For folds that are expensive, or violate InstCombine requirements. + * `VectorCombine`: For folds of vector operations that require target-dependent cost-modelling. + +Sometimes, folds that logically belong in InstSimplify are placed in InstCombine instead, for example because they are too expensive, or because they are structurally simpler to implement in InstCombine. + +For example, if a fold produces new instructions in some cases but returns an existing value in others, it may be preferable to keep all cases in InstCombine, rather than trying to split them among InstCombine and InstSimplify. + +### Canonicalization and target-independence + +InstCombine is a target-independent canonicalization pass. This means that it +tries to bring IR into a "canonical form" that other optimizations (both inside +and outside of InstCombine) can rely on. For this reason, the chosen canonical +form needs to be the same for all targets, and not depend on target-specific +cost modelling. + +In many cases, "canonicalization" and "optimization" coincide. For example, if +we convert `x * 2` into `x << 1`, this both makes the IR more canonical +(because there is now only one way to express the same operation, rather than +two) and faster (because shifts will usually have lower latency than +multiplies). + +However, there are also canonicalizations that don't serve any direct +optimization purpose. For example, InstCombine will canonicalize non-strict +predicates like `ule` to strict predicates like `ult`. `icmp ule i8 %x, 7` +becomes `icmp ult i8 %x, 8`. This is not an optimization in any meaningful +sense, but it does reduce the number of cases that other transforms need to +handle. + +If some canonicalization is not profitable for a specific target, then a reverse +transform needs to be added in the backend. Patches to disable specific +InstCombine transforms on certain targets, or to drive them using +target-specific cost-modelling, **will not be accepted**. The only permitted +target-dependence is on DataLayout and TargetLibraryInfo. + +The use of TargetTransformInfo is only allowed for hooks for target-specific +intrinsics, such as `TargetTransformInfo::instCombineIntrinsic()`. These are +already inherently target-dependent anyway. + +For vector-specific transforms that require cost-modelling, the VectorCombine +pass can be used instead. In very rare circumstances, if there are no other +alternatives, target-dependent transforms may be accepted into +AggressiveInstCombine. + +### PatternMatch + +Many transforms make use of the matching infrastructure defined in +[PatternMatch.h](https://github.com/llvm/llvm-project/blame/main/llvm/include/llvm/IR/PatternMatch.h). + +Here is a typical usage example: + +``` +// Fold (A - B) + B and B + (A - B) to A. +Value *A, *B; +if (match(V, m_c_Add(m_Sub(m_Value(A), m_Value(B)), m_Deferred(B)))) + return A; +``` + +And another: + +``` +// Fold A + C1 == C2 to A == C1+C2 +Value *A; +if (match(V, m_ICmp(Pred, m_Add(m_Value(A), m_APInt(C1)), m_APInt(C2))) && + ICmpInst::isEquality(Pred)) + return Builder.CreateICmp(Pred, A, + ConstantInt::get(A->getType(), *C1 + *C2)); +``` + +Some common matchers are: + + * `m_Value(A)`: Match any value and write it into `Value *A`. + * `m_Specific(A)`: Check that the operand equals A. Use this if A is + assigned **outside** the pattern. + * `m_Deferred(A)`: Check that the operand equals A. Use this if A is + assigned **inside** the pattern, for example via `m_Value(A)`. + * `m_APInt(C)`: Match a scalar integer constant or splat vector constant into + `const APInt *C`. Does not permit undef/poison values. + * `m_ImmConstant(C)`: Match any non-constant-expression constant into + `Constant *C`. + * `m_Constant(C)`: Match any constant into `Constant *C`. Don't use this unless + you know what you're doing. + * `m_Add(M1, M2)`, `m_Sub(M1, M2)`, etc: Match an add/sub/etc where the first + operand matches M1 and the second M2. + * `m_c_Add(M1, M2)`, etc: Match an add commutatively. The operands must match + either M1 and M2 or M2 and M1. Most instruction matchers have a commutative + variant. + * `m_ICmp(Pred, M1, M2)` and `m_c_ICmp(Pred, M1, M2)`: Match an icmp, writing + the predicate into `IcmpInst::Predicate Pred`. If the commutative version + is used, and the operands match in order M2, M1, then `Pred` will be the + swapped predicate. + * `m_OneUse(M)`: Check that the value only has one use, and also matches M. + For example `m_OneUse(m_Add(...))`. See the next section for more + information. + +See the header for the full list of available matchers. + +### InstCombine APIs + +InstCombine transforms are handled by `visitXYZ()` methods, where XYZ +corresponds to the root instruction of your transform. If the outermost +instruction of the pattern you are matching is an icmp, the fold will be +located somewhere inside `visitICmpInst()`. + +The return value of the visit method is an instruction. You can either return +a new instruction, in which case it will be inserted before the old one, and +uses of the old one will be replaced by it. Or you can return the original +instruction to indicate that *some* kind of change has been made. Finally, a +nullptr return value indicates that no change occurred. + +For example, if your transform produces a single new icmp instruction, you could +write the following: + +``` +if (...) + return new ICmpInst(Pred, X, Y); +``` + +In this case the main InstCombine loop takes care of inserting the instruction +and replacing uses of the old instruction. + +Alternatively, you can also write it like this: + +``` +if (...) + return replaceInstUsesWith(OrigI, Builder.CreateICmp(Pred, X, Y)); +``` + +In this case `IRBuilder` will insert the instruction and `replaceInstUsesWith()` +will replace the uses of the old instruction, and return it to indicate that +a change occurred. + +Both forms are equivalent, and you can use whichever is more convenient in +context. For example, it's common that folds are inside helper functions that +return `Value *` and then `replaceInstUsesWith()` is invoked on the result of +that helper. + +InstCombine makes use of a worklist, which needs to be correctly updated during +transforms. This usually happens automatically, but there are some things to +keep in mind: + + * Don't use the `Value::replaceAllUsesWith()` API. Use InstCombine's + `replaceInstUsesWith()` helper instead. + * Don't use the `Instruction::eraseFromParent()` API. Use InstCombine's + `eraseInstFromFunction()` helper instead. (Explicitly erasing instruction + is usually not necessary, as side-effect free instructions without users + are automatically removed.) + * Apart from the "directly return an instruction" pattern above, use IRBUilder + to create all instruction. Do not manually create and insert them. + * When replacing operands or uses of instructions, use `replaceOperand()` + and `replaceUse()` instead of `setOperand()`. + +### Multi-use handling + +Transforms should usually not increase the total number of instructions. This +is not a hard requirement: For example, it is usually worthwhile to replace a +single division instruction with multiple other instructions. + +For example, if you have a transform that replaces two instructions, with two +other instructions, this is (usually) only profitable if *both* the original +instructions can be removed. To ensure that both instructions are removed, you +need to add a one-use check for the inner instruction. + +One-use checks can be performed using the `m_OneUse()` matcher, or the +`V->hasOneUse()` method. + +### Generalization + +Transforms can both be too specific (only handling some odd subset of patterns, +leading to unexpected optimization cliffs) and too general (introducing +complexity to handle cases with no real-world relevance). The right level of +generality is quite subjective, so this section only provides some broad +guidelines. + + * Avoid transforms that are hardcoded to specific constants. Try to figure + out what the general rule for arbitrary constants is. + * Add handling for conjugate patterns. For example, if you implement a fold + for `icmp eq`, you almost certainly also want to support `icmp ne`, with the + inverse result. Similarly, if you implement a pattern for `and` of `icmp`s, + you should also handle the de-Morgan conjugate using `or`. + * Handle non-splat vector constants if doing so is free, but do not add + handling for them if it adds any additional complexity to the code. + * Do not handle non-canonical patterns, unless there is a specific motivation + to do so. For example, it may sometimes be worthwhile to handle a pattern + that would normally be converted into a different canonical form, but can + still occur in multi-use scenarios. This is fine to do if there is specific + real-world motivation, but you should not go out of your way to do this + otherwise. + * Sometimes the motivating pattern uses a constant value with certain + properties, but the fold can be generalized to non-constant values by making + use of ValueTracking queries. Whether this makes sense depends on the case, + but it's usually a good idea to only handle the constant pattern first, and + then generalize later if it seems useful. diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst index 155cb3361669..f40a04d414a2 100644 --- a/llvm/docs/UserGuides.rst +++ b/llvm/docs/UserGuides.rst @@ -43,6 +43,7 @@ intermediate LLVM representation. HowToCrossCompileBuiltinsOnArm HowToCrossCompileLLVM HowToUpdateDebugInfo + InstCombineContributorGuide InstrProfileFormat InstrRefDebugInfo LinkTimeOptimization @@ -186,6 +187,10 @@ Optimizations :doc:`InstrProfileFormat` This document explains two binary formats of instrumentation-based profiles. +:doc:`InstCombineContributorGuide` + This document specifies guidelines for contributions for InstCombine and + related passes. + Code Generation --------------- -- GitLab From 8d7a6e2fd8beadc54a2e54fa361d35c66fdb40a4 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 08:00:03 -0700 Subject: [PATCH 058/933] [SLP]Fix a crash for gather node with instructions from different bbs, if cost threshold is very low. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 11 ++++++--- .../X86/gather-nodes-different-bb.ll | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 4853c2006fea..36b446962c4a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9275,11 +9275,16 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { // Check if any of the gather node forms an insertelement buildvector // somewhere. - if (any_of(VectorizableTree, [](const std::unique_ptr &TE) { + bool IsAllowedSingleBVNode = + VectorizableTree.size() > 1 || + (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() && + allSameBlock(VectorizableTree.front()->Scalars)); + if (any_of(VectorizableTree, [&](const std::unique_ptr &TE) { return TE->State == TreeEntry::NeedToGather && - all_of(TE->Scalars, [](Value *V) { + all_of(TE->Scalars, [&](Value *V) { return isa(V) || - (!V->hasNUsesOrMore(UsesLimit) && + (IsAllowedSingleBVNode && + !V->hasNUsesOrMore(UsesLimit) && any_of(V->users(), [](User *U) { return isa(U); })); diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll new file mode 100644 index 000000000000..2acbe89b9775 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll @@ -0,0 +1,24 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr="-avx512pf,+avx512f,+avx512bw" -slp-threshold=-100 < %s | FileCheck %s + +define i1 @foo(i32 %a) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = sub nsw i32 0, [[A:%.*]] +; CHECK-NEXT: br label [[BB4:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[LOCAL:%.*]] = sub nsw i32 0, 0 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i32 0 +; CHECK-NEXT: [[ADD:%.*]] = icmp eq i32 [[TMP0]], [[LOCAL]] +; CHECK-NEXT: ret i1 [[ADD]] +; +entry: + %0 = sub nsw i32 0, %a + br label %bb1 + +bb1: + %local = sub nsw i32 0, 0 + %ins1 = insertelement <2 x i32> poison, i32 %0, i32 0 + %add = icmp eq i32 %0, %local + ret i1 %add +} -- GitLab From e4fa2e3562f20106f0fe4a4d09df03a548db4eae Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 15:11:59 +0000 Subject: [PATCH 059/933] [DAG] isGuaranteedNotToBeUndefOrPoisonForTargetNode - add fallback implementation (#86125) Allow targets to rely on TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode to test nodes for canCreateUndefOrPoisonForTargetNode + all arguments are isGuaranteedNotToBeUndefOrPoison. Targets can still perform this themselves for specific special case nodes (e.g. target shuffles). Matches the fallback in SelectionDAG::isGuaranteedNotToBeUndefOrPoison --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 5 +++-- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 9d73a42df2a4..cd6f083243d0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5042,8 +5042,9 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, // If Op can't create undef/poison and none of its operands are undef/poison // then Op is never undef/poison. - // NOTE: TargetNodes should handle this in themselves in - // isGuaranteedNotToBeUndefOrPoisonForTargetNode. + // NOTE: TargetNodes can handle this in themselves in + // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let + // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it. return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true, Depth) && all_of(Op->ops(), [&](SDValue V) { diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 57f8fc409de4..da29b1d5b312 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3786,7 +3786,15 @@ bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op" " is a target node!"); - return false; + + // If Op can't create undef/poison and none of its operands are undef/poison + // then Op is never undef/poison. + return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, PoisonOnly, + /*ConsiderFlags*/ true, Depth) && + all_of(Op->ops(), [&](SDValue V) { + return DAG.isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, + Depth + 1); + }); } bool TargetLowering::canCreateUndefOrPoisonForTargetNode( -- GitLab From 7678e6e562811688b472ad19900fa64cd00b7c06 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 08:14:48 -0700 Subject: [PATCH 060/933] [RISCV] Lower the alignment requirement for a GPR pair spill for Zdinx on RV32. (#85871) I believe we can use XLen alignment as long as eliminateFrameIndex limits the maximum folded offset to 2043. This way when we split the load/store into two 2 instructions we'll be able to add 4 without overflowing simm12. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 7 ++ llvm/lib/Target/RISCV/RISCVRegisterInfo.td | 2 +- llvm/test/CodeGen/RISCV/zdinx-large-spill.mir | 74 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/zdinx-large-spill.mir diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 952d17468da5..74d65324b95d 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -446,6 +446,13 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, (Lo12 & 0b11111) != 0) { // Prefetch instructions require the offset to be 32 byte aligned. MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0); + } else if ((Opc == RISCV::PseudoRV32ZdinxLD || + Opc == RISCV::PseudoRV32ZdinxSD) && + Lo12 >= 2044) { + // This instruction will be split into 2 instructions. The second + // instruction will add 4 to the immediate. If that would overflow 12 + // bits, we can't fold the offset. + MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0); } else { // We can encode an add with 12 bit signed immediate in the immediate // operand of our user instruction. As a result, the remaining diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index 225b57554c1d..9da1f73681c6 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -573,7 +573,7 @@ let RegAltNameIndices = [ABIRegAltName] in { } let RegInfos = RegInfoByHwMode<[RV32, RV64], - [RegInfo<64, 64, 64>, RegInfo<128, 128, 128>]>, + [RegInfo<64, 64, 32>, RegInfo<128, 128, 64>]>, DecoderMethod = "DecodeGPRPairRegisterClass" in def GPRPair : RegisterClass<"RISCV", [XLenPairFVT], 64, (add X10_X11, X12_X13, X14_X15, X16_X17, diff --git a/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir new file mode 100644 index 000000000000..a2a722a7f728 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir @@ -0,0 +1,74 @@ +# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +# RUN: llc %s -mtriple=riscv32 -mattr=+zdinx -start-before=prologepilog -o - | FileCheck %s + +# We want to make sure eliminateFrameIndex doesn't fold sp+2044 as an offset in +# a GPR pair spill/reload instruction. When we split the pair spill, we would be +# unable to add 4 to the immediate without overflowing simm12. + +--- | + define void @foo() { + ; CHECK-LABEL: foo: + ; CHECK: # %bb.0: + ; CHECK-NEXT: addi sp, sp, -2048 + ; CHECK-NEXT: addi sp, sp, -16 + ; CHECK-NEXT: .cfi_def_cfa_offset 2064 + ; CHECK-NEXT: lui t0, 1 + ; CHECK-NEXT: add t0, sp, t0 + ; CHECK-NEXT: sw a0, -2040(t0) + ; CHECK-NEXT: sw a1, -2036(t0) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: sw a2, -2048(a0) + ; CHECK-NEXT: sw a3, -2044(a0) + ; CHECK-NEXT: sw a4, 2040(sp) + ; CHECK-NEXT: sw a5, 2044(sp) + ; CHECK-NEXT: sw a6, 2032(sp) + ; CHECK-NEXT: sw a7, 2036(sp) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: lw a1, -2036(a0) + ; CHECK-NEXT: lw a0, -2040(a0) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: lw a2, -2048(a0) + ; CHECK-NEXT: lw a3, -2044(a0) + ; CHECK-NEXT: lw a4, 2040(sp) + ; CHECK-NEXT: lw a5, 2044(sp) + ; CHECK-NEXT: lw a6, 2032(sp) + ; CHECK-NEXT: lw a7, 2036(sp) + ; CHECK-NEXT: addi sp, sp, 2032 + ; CHECK-NEXT: addi sp, sp, 32 + ; CHECK-NEXT: ret + ret void + } +... +--- +name: foo +tracksRegLiveness: true +tracksDebugUserValues: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 8, alignment: 4 } + - { id: 1, type: spill-slot, size: 8, alignment: 4 } + - { id: 2, type: spill-slot, size: 8, alignment: 4 } + - { id: 3, type: spill-slot, size: 8, alignment: 4 } + - { id: 4, type: spill-slot, size: 2024, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: 0 + varArgsSaveSize: 0 +body: | + bb.0: + liveins: $x10_x11, $x12_x13, $x14_x15, $x16_x17 + + PseudoRV32ZdinxSD killed renamable $x10_x11, %stack.0, 0 :: (store (s64) into %stack.0, align 4) + PseudoRV32ZdinxSD killed renamable $x12_x13, %stack.1, 0 :: (store (s64) into %stack.1, align 4) + PseudoRV32ZdinxSD killed renamable $x14_x15, %stack.2, 0 :: (store (s64) into %stack.2, align 4) + PseudoRV32ZdinxSD killed renamable $x16_x17, %stack.3, 0 :: (store (s64) into %stack.3, align 4) + renamable $x10_x11 = PseudoRV32ZdinxLD %stack.0, 0 :: (load (s64) from %stack.0, align 4) + renamable $x12_x13 = PseudoRV32ZdinxLD %stack.1, 0 :: (load (s64) from %stack.1, align 4) + renamable $x14_x15 = PseudoRV32ZdinxLD %stack.2, 0 :: (load (s64) from %stack.2, align 4) + renamable $x16_x17 = PseudoRV32ZdinxLD %stack.3, 0 :: (load (s64) from %stack.3, align 4) + PseudoRET + +... -- GitLab From c04807c84e2a2653ab325f1b8ec73916565e6c54 Mon Sep 17 00:00:00 2001 From: aniplcc Date: Thu, 21 Mar 2024 20:51:06 +0530 Subject: [PATCH 061/933] [libc][c11] Add stdio.h's rename() function (#85068) Adds stdio.h's rename() function as defined in n3096. Fixes #84980. --- libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/docs/stdio.rst | 2 +- libc/spec/stdc.td | 5 ++ libc/src/stdio/CMakeLists.txt | 7 +++ libc/src/stdio/linux/CMakeLists.txt | 12 +++++ libc/src/stdio/linux/rename.cpp | 26 ++++++++++ libc/src/stdio/rename.h | 18 +++++++ libc/test/src/stdio/CMakeLists.txt | 15 ++++++ libc/test/src/stdio/rename_test.cpp | 49 +++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 11 +++++ 12 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 libc/src/stdio/linux/rename.cpp create mode 100644 libc/src/stdio/rename.h create mode 100644 libc/test/src/stdio/rename_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index dbf81c284e78..6abb35ab0ead 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -195,6 +195,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.vsprintf diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index b42a55a4d712..e34c87ec6b5d 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -196,6 +196,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.fprintf diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index c216f4349627..8e1ab5cd65f0 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -198,6 +198,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.fprintf diff --git a/libc/docs/stdio.rst b/libc/docs/stdio.rst index 4fd6b71a0917..d17821562c25 100644 --- a/libc/docs/stdio.rst +++ b/libc/docs/stdio.rst @@ -68,7 +68,7 @@ These functions operate on files on the host's system, without using the Function_Name Available ============= ========= remove |check| -rename +rename |check| tmpnam ============= ========= diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 920036adfed5..76010a4b4533 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -706,6 +706,11 @@ def StdC : StandardSpec<"stdc"> { RetValSpec, [ArgSpec] >, + FunctionSpec< + "rename", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "setbuf", RetValSpec, diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt index ece93fd56ef0..11e15c917351 100644 --- a/libc/src/stdio/CMakeLists.txt +++ b/libc/src/stdio/CMakeLists.txt @@ -256,6 +256,13 @@ add_entrypoint_object( .${LIBC_TARGET_OS}.remove ) +add_entrypoint_object( + rename + ALIAS + DEPENDS + .${LIBC_TARGET_OS}.rename +) + # These entrypoints have multiple potential implementations. add_stdio_entrypoint_object(feof) add_stdio_entrypoint_object(feof_unlocked) diff --git a/libc/src/stdio/linux/CMakeLists.txt b/libc/src/stdio/linux/CMakeLists.txt index 774f24b2db0b..a08ff0ba4832 100644 --- a/libc/src/stdio/linux/CMakeLists.txt +++ b/libc/src/stdio/linux/CMakeLists.txt @@ -12,3 +12,15 @@ add_entrypoint_object( libc.src.__support.OSUtil.osutil libc.src.errno.errno ) + +add_entrypoint_object( + rename + SRCS + rename.cpp + HDRS + ../rename.h + DEPENDS + libc.include.sys_syscall + libc.src.__support.OSUtil.osutil + libc.src.errno.errno +) diff --git a/libc/src/stdio/linux/rename.cpp b/libc/src/stdio/linux/rename.cpp new file mode 100644 index 000000000000..f3d684249ad2 --- /dev/null +++ b/libc/src/stdio/linux/rename.cpp @@ -0,0 +1,26 @@ +//===-- Linux implementation of rename ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdio/rename.h" +#include "src/__support/OSUtil/syscall.h" // For internal syscall function. +#include "src/__support/common.h" +#include "src/errno/libc_errno.h" +#include // For syscall numbers. + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, rename, (const char *oldpath, const char *newpath)) { + int ret = LIBC_NAMESPACE::syscall_impl(SYS_rename, oldpath, newpath); + + if (ret >= 0) + return 0; + libc_errno = -ret; + return -1; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdio/rename.h b/libc/src/stdio/rename.h new file mode 100644 index 000000000000..eadda7c3eac9 --- /dev/null +++ b/libc/src/stdio/rename.h @@ -0,0 +1,18 @@ +//===-- Implementation header of rename -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDIO_RENAME_H +#define LLVM_LIBC_SRC_STDIO_RENAME_H + +namespace LIBC_NAMESPACE { + +int rename(const char *oldpath, const char *newpath); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDIO_RENAME_H diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index 3ccce16a76a2..a11a232c27e1 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -354,6 +354,21 @@ if(${LIBC_TARGET_OS} STREQUAL "linux") libc.src.unistd.access libc.src.unistd.close ) + + add_libc_test( + rename_test + SUITE + libc_stdio_unittests + SRCS + rename_test.cpp + DEPENDS + libc.src.errno.errno + libc.src.fcntl.open + libc.src.stdio.rename + libc.src.unistd.access + libc.src.unistd.close + libc.test.UnitTest.ErrnoSetterMatcher + ) endif() add_libc_test( diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp new file mode 100644 index 000000000000..a9fbe24ded9c --- /dev/null +++ b/libc/test/src/stdio/rename_test.cpp @@ -0,0 +1,49 @@ +//===-- Unittests for rename ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/errno/libc_errno.h" +#include "src/fcntl/open.h" +#include "src/stdio/rename.h" +#include "src/unistd/access.h" +#include "src/unistd/close.h" +#include "test/UnitTest/ErrnoSetterMatcher.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcRenameTest, CreateAndRenameFile) { + // The test strategy is to create a file and rename it, and also verify that + // it was renamed. + LIBC_NAMESPACE::libc_errno = 0; + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails; + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds; + + constexpr const char *FILENAME0 = "rename.test.file0"; + auto TEST_FILEPATH0 = libc_make_test_file_path(FILENAME0); + + int fd = LIBC_NAMESPACE::open(TEST_FILEPATH0, O_WRONLY | O_CREAT, S_IRWXU); + ASSERT_ERRNO_SUCCESS(); + ASSERT_GT(fd, 0); + ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH0, F_OK), Succeeds(0)); + + constexpr const char *FILENAME1 = "rename.test.file1"; + auto TEST_FILEPATH1 = libc_make_test_file_path(FILENAME1); + ASSERT_THAT(LIBC_NAMESPACE::rename(TEST_FILEPATH0, TEST_FILEPATH1), + Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH1, F_OK), Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH0, F_OK), Fails(ENOENT)); +} + +TEST(LlvmLibcRenameTest, RenameNonExistent) { + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails; + + constexpr const char *FILENAME1 = "rename.test.file1"; + auto TEST_FILEPATH1 = libc_make_test_file_path(FILENAME1); + + ASSERT_THAT(LIBC_NAMESPACE::rename("non-existent", TEST_FILEPATH1), + Fails(ENOENT)); +} diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 40cfb1f470db..fe4b8e2de14e 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -3262,6 +3262,17 @@ libc_function( ], ) +libc_function( + name = "rename", + srcs = ["src/stdio/linux/rename.cpp"], + hdrs = ["src/stdio/rename.h"], + deps = [ + ":__support_common", + ":__support_osutil_syscall", + ":errno", + ], +) + ############################### sys/stat targets ############################### libc_function( -- GitLab From 50801f1095d33e712c3a51fdeef82569bd09007f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 21 Mar 2024 17:29:15 +0200 Subject: [PATCH 062/933] Reapply [libcxx] [modules] Fix relative paths with absolute LIBCXX_INSTALL_MODULES_DIR (#86020) This reapplies 272d1b44efdedb68c194970a610f0ca1b7b769c5 (from #85756), which was reverted in 407937036fa7640f61f225474b1ea6623a40dbdd. In the previous attempt, empty CMAKE_INSTALL_PREFIX was handled by quoting them, in d209d1340b99d4fbd325dffb5e13b757ab8264ea. That made the calls to cmake_path(ABSOLUTE_PATH) succeed, but the output paths of that weren't actually absolute, which was required by file(RELATIVE_PATH). Avoid this issue by constructing a non-empty base directory variable to use for calculating the relative path. --- libcxx/modules/CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 0dea8cfca94a..d47d19a47553 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -206,9 +206,20 @@ add_custom_target(generate-cxx-modules # Configure the modules manifest. # Use the relative path between the installation and the module in the json # file. This allows moving the entire installation to a different location. +if("${CMAKE_INSTALL_PREFIX}" STREQUAL "") + set(BASE_DIRECTORY "/") +else() + set(BASE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +endif() +cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_LIBRARY_DIR + BASE_DIRECTORY ${BASE_DIRECTORY} + OUTPUT_VARIABLE ABS_LIBRARY_DIR) +cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_MODULES_DIR + BASE_DIRECTORY ${BASE_DIRECTORY} + OUTPUT_VARIABLE ABS_MODULES_DIR) file(RELATIVE_PATH LIBCXX_MODULE_RELATIVE_PATH - ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_LIBRARY_DIR} - ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_MODULES_DIR}) + ${ABS_LIBRARY_DIR} + ${ABS_MODULES_DIR}) configure_file( "modules.json.in" "${LIBCXX_LIBRARY_DIR}/libc++.modules.json" -- GitLab From 7650a01927b8488b1d6d0930109e78c695193faf Mon Sep 17 00:00:00 2001 From: timoh-ba Date: Thu, 21 Mar 2024 16:30:10 +0100 Subject: [PATCH 063/933] [DWARF5][COFF] Emit section-relative .debug_line_str relocations (#83773) Dwarf 5 allows separating filenames from .debug_line into a separate .debug_line_str section. The strings are referenced relative to the start of the .debug_line_str section. Previously, on COFF, the relocation information instead caused offsets to be relocated to the base address of the COFF-File. This lead to wrong offsets in linked COFF (PE) files which caused the debugger to be unable to find the correct source files. This patch fixes this problem by making the offsets relative to the start of the .debug_line_str section instead. There should be no changes for ELF-Files as everything seems to be working there. A test is also added to ensure that the correct relocation entries are emitted. --- llvm/lib/MC/MCDwarf.cpp | 7 ++++++- llvm/test/MC/COFF/dwarf5lineinfo.s | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 llvm/test/MC/COFF/dwarf5lineinfo.s diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp index d0face9140de..2ee0c3eb27b9 100644 --- a/llvm/lib/MC/MCDwarf.cpp +++ b/llvm/lib/MC/MCDwarf.cpp @@ -360,7 +360,12 @@ void MCDwarfLineStr::emitRef(MCStreamer *MCOS, StringRef Path) { size_t Offset = addString(Path); if (UseRelocs) { MCContext &Ctx = MCOS->getContext(); - MCOS->emitValue(makeStartPlusIntExpr(Ctx, *LineStrLabel, Offset), RefSize); + if (Ctx.getAsmInfo()->needsDwarfSectionOffsetDirective()) { + MCOS->emitCOFFSecRel32(LineStrLabel, Offset); + } else { + MCOS->emitValue(makeStartPlusIntExpr(Ctx, *LineStrLabel, Offset), + RefSize); + } } else MCOS->emitIntValue(Offset, RefSize); } diff --git a/llvm/test/MC/COFF/dwarf5lineinfo.s b/llvm/test/MC/COFF/dwarf5lineinfo.s new file mode 100644 index 000000000000..f0789feb2085 --- /dev/null +++ b/llvm/test/MC/COFF/dwarf5lineinfo.s @@ -0,0 +1,13 @@ +// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-gnu %s -o - | llvm-readobj -r - | FileCheck %s + +// CHECK: Relocations [ +// CHECK: Section (4) .debug_line { +// CHECK: 0x22 IMAGE_REL_AMD64_SECREL .debug_line_str (8) +// CHECK: 0x2C IMAGE_REL_AMD64_SECREL .debug_line_str (8) +// CHECK: 0x36 IMAGE_REL_AMD64_ADDR64 .text (0) +// CHECK: } + +main: + .file 0 "/" "test.c" + .loc 0 1 0 + retq -- GitLab From 0c8dfc85c3740bd8905e21642f616e6fd54854e0 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 21 Mar 2024 08:36:47 -0700 Subject: [PATCH 064/933] [libc][stdio][test] fixup rename test (#86136) Link: #84980 Link: #85068 --- libc/test/src/stdio/CMakeLists.txt | 2 +- libc/test/src/stdio/rename_test.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index a11a232c27e1..4c38e8aba7d7 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -354,7 +354,7 @@ if(${LIBC_TARGET_OS} STREQUAL "linux") libc.src.unistd.access libc.src.unistd.close ) - + add_libc_test( rename_test SUITE diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp index a9fbe24ded9c..3ed39fe8c0eb 100644 --- a/libc/test/src/stdio/rename_test.cpp +++ b/libc/test/src/stdio/rename_test.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "include/llvm-libc-macros/linux/unistd-macros.h" +#include "include/llvm-libc-macros/linux/sys-stat-macros.h" #include "src/errno/libc_errno.h" #include "src/fcntl/open.h" #include "src/stdio/rename.h" -- GitLab From 556fe5f290ea88dcbb7ced16b0f057dcebce1fd0 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 21 Mar 2024 08:40:08 -0700 Subject: [PATCH 065/933] [lldb] Reland: Store SupportFile in FileEntry (NFC) (#85892) This is another step towards supporting DWARF5 checksums and inline source code in LLDB. This is a reland of #85468 but without the functional change of storing the support file from the line table (yet). --- lldb/include/lldb/Core/Disassembler.h | 2 +- lldb/include/lldb/Symbol/LineEntry.h | 5 ++++- lldb/include/lldb/Utility/SupportFile.h | 3 +++ lldb/source/API/SBLineEntry.cpp | 10 +++++----- lldb/source/API/SBThread.cpp | 2 +- lldb/source/Breakpoint/BreakpointResolver.cpp | 2 +- .../Breakpoint/BreakpointResolverFileLine.cpp | 7 ++++--- lldb/source/Commands/CommandObjectBreakpoint.cpp | 4 ++-- lldb/source/Commands/CommandObjectSource.cpp | 14 +++++++------- lldb/source/Commands/CommandObjectThread.cpp | 2 +- lldb/source/Core/Address.cpp | 2 +- lldb/source/Core/Disassembler.cpp | 8 ++++---- lldb/source/Core/FormatEntity.cpp | 2 +- lldb/source/Core/IOHandlerCursesGUI.cpp | 11 ++++++----- lldb/source/Core/SourceManager.cpp | 2 +- .../Clang/ClangExpressionSourceCode.cpp | 2 +- lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp | 4 ++-- lldb/source/Symbol/CompileUnit.cpp | 2 +- lldb/source/Symbol/Function.cpp | 4 ++-- lldb/source/Symbol/LineEntry.cpp | 13 +++++++------ lldb/source/Symbol/LineTable.cpp | 4 ++-- lldb/source/Symbol/SymbolContext.cpp | 4 ++-- lldb/source/Target/StackFrame.cpp | 2 +- lldb/source/Target/StackFrameList.cpp | 4 ++-- lldb/source/Target/Thread.cpp | 8 ++++---- lldb/source/Target/TraceDumper.cpp | 4 ++-- .../SymbolFile/PDB/SymbolFilePDBTests.cpp | 2 +- 27 files changed, 69 insertions(+), 60 deletions(-) diff --git a/lldb/include/lldb/Core/Disassembler.h b/lldb/include/lldb/Core/Disassembler.h index 885ac1bb4a7e..e037a49f152c 100644 --- a/lldb/include/lldb/Core/Disassembler.h +++ b/lldb/include/lldb/Core/Disassembler.h @@ -538,7 +538,7 @@ protected: ElideMixedSourceAndDisassemblyLine(const ExecutionContext &exe_ctx, const SymbolContext &sc, LineEntry &line) { SourceLine sl; - sl.file = line.file; + sl.file = line.GetFile(); sl.line = line.line; sl.column = line.column; return ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, sl); diff --git a/lldb/include/lldb/Symbol/LineEntry.h b/lldb/include/lldb/Symbol/LineEntry.h index 31e1cd0b36f9..8da59cf0bd24 100644 --- a/lldb/include/lldb/Symbol/LineEntry.h +++ b/lldb/include/lldb/Symbol/LineEntry.h @@ -130,11 +130,14 @@ struct LineEntry { /// Shared pointer to the target this LineEntry belongs to. void ApplyFileMappings(lldb::TargetSP target_sp); + /// Helper to access the file. + const FileSpec &GetFile() const { return file_sp->GetSpecOnly(); } + /// The section offset address range for this line entry. AddressRange range; /// The source file, possibly mapped by the target.source-map setting. - FileSpec file; + lldb::SupportFileSP file_sp; /// The original source file, from debug info. lldb::SupportFileSP original_file_sp; diff --git a/lldb/include/lldb/Utility/SupportFile.h b/lldb/include/lldb/Utility/SupportFile.h index 0ea0ca4e7c97..7505d7f345c5 100644 --- a/lldb/include/lldb/Utility/SupportFile.h +++ b/lldb/include/lldb/Utility/SupportFile.h @@ -45,6 +45,9 @@ public: /// Materialize the file to disk and return the path to that temporary file. virtual const FileSpec &Materialize() { return m_file_spec; } + /// Change the file name. + void Update(const FileSpec &file_spec) { m_file_spec = file_spec; } + protected: FileSpec m_file_spec; Checksum m_checksum; diff --git a/lldb/source/API/SBLineEntry.cpp b/lldb/source/API/SBLineEntry.cpp index 28d12e65fdaf..99a7b8fe644c 100644 --- a/lldb/source/API/SBLineEntry.cpp +++ b/lldb/source/API/SBLineEntry.cpp @@ -81,8 +81,8 @@ SBFileSpec SBLineEntry::GetFileSpec() const { LLDB_INSTRUMENT_VA(this); SBFileSpec sb_file_spec; - if (m_opaque_up.get() && m_opaque_up->file) - sb_file_spec.SetFileSpec(m_opaque_up->file); + if (m_opaque_up.get() && m_opaque_up->GetFile()) + sb_file_spec.SetFileSpec(m_opaque_up->GetFile()); return sb_file_spec; } @@ -109,9 +109,9 @@ void SBLineEntry::SetFileSpec(lldb::SBFileSpec filespec) { LLDB_INSTRUMENT_VA(this, filespec); if (filespec.IsValid()) - ref().file = filespec.ref(); + ref().file_sp = std::make_shared(filespec.ref()); else - ref().file.Clear(); + ref().file_sp = std::make_shared(); } void SBLineEntry::SetLine(uint32_t line) { LLDB_INSTRUMENT_VA(this, line); @@ -168,7 +168,7 @@ bool SBLineEntry::GetDescription(SBStream &description) { if (m_opaque_up) { char file_path[PATH_MAX * 2]; - m_opaque_up->file.GetPath(file_path, sizeof(file_path)); + m_opaque_up->GetFile().GetPath(file_path, sizeof(file_path)); strm.Printf("%s:%u", file_path, GetLine()); if (GetColumn() > 0) strm.Printf(":%u", GetColumn()); diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp index fa4c80e59d97..eb9cf063802c 100644 --- a/lldb/source/API/SBThread.cpp +++ b/lldb/source/API/SBThread.cpp @@ -819,7 +819,7 @@ SBError SBThread::StepOverUntil(lldb::SBFrame &sb_frame, step_file_spec = sb_file_spec.ref(); } else { if (frame_sc.line_entry.IsValid()) - step_file_spec = frame_sc.line_entry.file; + step_file_spec = frame_sc.line_entry.GetFile(); else { sb_error.SetErrorString("invalid file argument or no file for frame"); return sb_error; diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp index 1861a0fe7c4f..ff4e2a998519 100644 --- a/lldb/source/Breakpoint/BreakpointResolver.cpp +++ b/lldb/source/Breakpoint/BreakpointResolver.cpp @@ -221,7 +221,7 @@ void BreakpointResolver::SetSCMatchesByLine( auto &match = all_scs[0]; auto worklist_begin = std::partition( all_scs.begin(), all_scs.end(), [&](const SymbolContext &sc) { - if (sc.line_entry.file == match.line_entry.file || + if (sc.line_entry.GetFile() == match.line_entry.GetFile() || *sc.line_entry.original_file_sp == *match.line_entry.original_file_sp) { // When a match is found, keep track of the smallest line number. diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp index cc4e1d26724f..d7d8c714867e 100644 --- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp @@ -147,8 +147,9 @@ void BreakpointResolverFileLine::FilterContexts(SymbolContextList &sc_list) { else continue; - if (file != sc.line_entry.file) { - LLDB_LOG(log, "unexpected symbol context file {0}", sc.line_entry.file); + if (file != sc.line_entry.GetFile()) { + LLDB_LOG(log, "unexpected symbol context file {0}", + sc.line_entry.GetFile()); continue; } @@ -223,7 +224,7 @@ void BreakpointResolverFileLine::DeduceSourceMapping( const bool case_sensitive = request_file.IsCaseSensitive(); for (const SymbolContext &sc : sc_list) { - FileSpec sc_file = sc.line_entry.file; + FileSpec sc_file = sc.line_entry.GetFile(); if (FileSpec::Equal(sc_file, request_file, /*full*/ true)) continue; diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp index fbece865f113..cd4c7790f447 100644 --- a/lldb/source/Commands/CommandObjectBreakpoint.cpp +++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp @@ -780,8 +780,8 @@ private: } else { const SymbolContext &sc = cur_frame->GetSymbolContext(eSymbolContextLineEntry); - if (sc.line_entry.file) { - file = sc.line_entry.file; + if (sc.line_entry.GetFile()) { + file = sc.line_entry.GetFile(); } else { result.AppendError("Can't find the file for the selected frame to " "use as the default file."); diff --git a/lldb/source/Commands/CommandObjectSource.cpp b/lldb/source/Commands/CommandObjectSource.cpp index fde74f02aea6..0c1267456a18 100644 --- a/lldb/source/Commands/CommandObjectSource.cpp +++ b/lldb/source/Commands/CommandObjectSource.cpp @@ -158,7 +158,7 @@ protected: if (module_list.GetSize() && module_list.GetIndexForModule(module) == LLDB_INVALID_INDEX32) continue; - if (!FileSpec::Match(file_spec, line_entry.file)) + if (!FileSpec::Match(file_spec, line_entry.GetFile())) continue; if (start_line > 0 && line_entry.line < start_line) continue; @@ -239,7 +239,7 @@ protected: num_matches++; if (num_lines > 0 && num_matches > num_lines) break; - assert(cu_file_spec == line_entry.file); + assert(cu_file_spec == line_entry.GetFile()); if (!cu_header_printed) { if (num_matches > 0) strm << "\n\n"; @@ -760,11 +760,11 @@ protected: bool operator<(const SourceInfo &rhs) const { if (function.GetCString() < rhs.function.GetCString()) return true; - if (line_entry.file.GetDirectory().GetCString() < - rhs.line_entry.file.GetDirectory().GetCString()) + if (line_entry.GetFile().GetDirectory().GetCString() < + rhs.line_entry.GetFile().GetDirectory().GetCString()) return true; - if (line_entry.file.GetFilename().GetCString() < - rhs.line_entry.file.GetFilename().GetCString()) + if (line_entry.GetFile().GetFilename().GetCString() < + rhs.line_entry.GetFile().GetFilename().GetCString()) return true; if (line_entry.line < rhs.line_entry.line) return true; @@ -799,7 +799,7 @@ protected: sc.function->GetEndLineSourceInfo(end_file, end_line); } else { // We have an inlined function - start_file = source_info.line_entry.file; + start_file = source_info.line_entry.GetFile(); start_line = source_info.line_entry.line; end_line = start_line + m_options.num_lines; } diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp index cf4f8ccaa0c4..3dbbfd4f9d34 100644 --- a/lldb/source/Commands/CommandObjectThread.cpp +++ b/lldb/source/Commands/CommandObjectThread.cpp @@ -1705,7 +1705,7 @@ protected: line = sym_ctx.line_entry.line + m_options.m_line_offset; // Try the current file, but override if asked. - FileSpec file = sym_ctx.line_entry.file; + FileSpec file = sym_ctx.line_entry.GetFile(); if (m_options.m_filenames.GetSize() == 1) file = m_options.m_filenames.GetFileSpecAtIndex(0); diff --git a/lldb/source/Core/Address.cpp b/lldb/source/Core/Address.cpp index 6f5c366ab38a..b23398883fa5 100644 --- a/lldb/source/Core/Address.cpp +++ b/lldb/source/Core/Address.cpp @@ -398,7 +398,7 @@ bool Address::GetDescription(Stream &s, Target &target, "Non-brief descriptions not implemented"); LineEntry line_entry; if (CalculateSymbolContextLineEntry(line_entry)) { - s.Printf(" (%s:%u:%u)", line_entry.file.GetFilename().GetCString(), + s.Printf(" (%s:%u:%u)", line_entry.GetFile().GetFilename().GetCString(), line_entry.line, line_entry.column); return true; } diff --git a/lldb/source/Core/Disassembler.cpp b/lldb/source/Core/Disassembler.cpp index 7b07fcb26813..e31746fa0b8b 100644 --- a/lldb/source/Core/Disassembler.cpp +++ b/lldb/source/Core/Disassembler.cpp @@ -201,7 +201,7 @@ Disassembler::GetFunctionDeclLineEntry(const SymbolContext &sc) { uint32_t func_decl_line; sc.function->GetStartLineSourceInfo(func_decl_file, func_decl_line); - if (func_decl_file != prologue_end_line.file && + if (func_decl_file != prologue_end_line.GetFile() && func_decl_file != prologue_end_line.original_file_sp->GetSpecOnly()) return {}; @@ -354,7 +354,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, } if (sc.line_entry.IsValid()) { SourceLine this_line; - this_line.file = sc.line_entry.file; + this_line.file = sc.line_entry.GetFile(); this_line.line = sc.line_entry.line; this_line.column = sc.line_entry.column; if (!ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, this_line)) @@ -406,7 +406,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, uint32_t func_decl_line; sc.function->GetStartLineSourceInfo(func_decl_file, func_decl_line); - if (func_decl_file == prologue_end_line.file || + if (func_decl_file == prologue_end_line.GetFile() || func_decl_file == prologue_end_line.original_file_sp->GetSpecOnly()) { // Add all the lines between the function declaration and @@ -439,7 +439,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, if (sc != prev_sc && sc.comp_unit && sc.line_entry.IsValid()) { SourceLine this_line; - this_line.file = sc.line_entry.file; + this_line.file = sc.line_entry.GetFile(); this_line.line = sc.line_entry.line; if (!ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp index cf82676bedda..ba62e2625259 100644 --- a/lldb/source/Core/FormatEntity.cpp +++ b/lldb/source/Core/FormatEntity.cpp @@ -1792,7 +1792,7 @@ bool FormatEntity::Format(const Entry &entry, Stream &s, if (sc && sc->line_entry.IsValid()) { Module *module = sc->module_sp.get(); if (module) { - if (DumpFile(s, sc->line_entry.file, (FileKind)entry.number)) + if (DumpFile(s, sc->line_entry.GetFile(), (FileKind)entry.number)) return true; } } diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp index f86dce247135..d922d32f9105 100644 --- a/lldb/source/Core/IOHandlerCursesGUI.cpp +++ b/lldb/source/Core/IOHandlerCursesGUI.cpp @@ -6894,7 +6894,8 @@ public: if (context_changed) m_selected_line = m_pc_line; - if (m_file_sp && m_file_sp->GetFileSpec() == m_sc.line_entry.file) { + if (m_file_sp && + m_file_sp->GetFileSpec() == m_sc.line_entry.GetFile()) { // Same file, nothing to do, we should either have the lines or // not (source file missing) if (m_selected_line >= static_cast(m_first_visible_line)) { @@ -6909,8 +6910,8 @@ public: } else { // File changed, set selected line to the line with the PC m_selected_line = m_pc_line; - m_file_sp = - m_debugger.GetSourceManager().GetFile(m_sc.line_entry.file); + m_file_sp = m_debugger.GetSourceManager().GetFile( + m_sc.line_entry.GetFile()); if (m_file_sp) { const size_t num_lines = m_file_sp->GetNumLines(); m_line_width = 1; @@ -7000,7 +7001,7 @@ public: LineEntry bp_loc_line_entry; if (bp_loc_sp->GetAddress().CalculateSymbolContextLineEntry( bp_loc_line_entry)) { - if (m_file_sp->GetFileSpec() == bp_loc_line_entry.file) { + if (m_file_sp->GetFileSpec() == bp_loc_line_entry.GetFile()) { bp_lines.insert(bp_loc_line_entry.line); } } @@ -7477,7 +7478,7 @@ public: LineEntry bp_loc_line_entry; if (bp_loc_sp->GetAddress().CalculateSymbolContextLineEntry( bp_loc_line_entry)) { - if (m_file_sp->GetFileSpec() == bp_loc_line_entry.file && + if (m_file_sp->GetFileSpec() == bp_loc_line_entry.GetFile() && m_selected_line + 1 == bp_loc_line_entry.line) { bool removed = exe_ctx.GetTargetRef().RemoveBreakpointByID(bp_sp->GetID()); diff --git a/lldb/source/Core/SourceManager.cpp b/lldb/source/Core/SourceManager.cpp index 517a4b0268d2..0d70c554e534 100644 --- a/lldb/source/Core/SourceManager.cpp +++ b/lldb/source/Core/SourceManager.cpp @@ -418,7 +418,7 @@ bool SourceManager::GetDefaultFileAndLine(FileSpec &file_spec, uint32_t &line) { if (sc.function->GetAddressRange() .GetBaseAddress() .CalculateSymbolContextLineEntry(line_entry)) { - SetDefaultFileAndLine(line_entry.file, line_entry.line); + SetDefaultFileAndLine(line_entry.GetFile(), line_entry.line); file_spec = m_last_file_spec; line = m_last_line; return true; diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp index 3d43ed3f99ff..3b601726388d 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp @@ -417,7 +417,7 @@ bool ClangExpressionSourceCode::GetText( if (sc.comp_unit && sc.line_entry.IsValid()) { DebugMacros *dm = sc.comp_unit->GetDebugMacros(); if (dm) { - AddMacroState state(sc.line_entry.file, sc.line_entry.line); + AddMacroState state(sc.line_entry.GetFile(), sc.line_entry.line); AddMacros(dm, sc.comp_unit, state, debug_macros_stream); } } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp index 10a1fe039189..bcb04fae15bd 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp @@ -106,13 +106,13 @@ bool lldb_private::formatters::LibcxxFunctionSummaryProvider( case CPPLanguageRuntime::LibCppStdFunctionCallableCase::Lambda: stream.Printf( " Lambda in File %s at Line %u", - callable_info.callable_line_entry.file.GetFilename().GetCString(), + callable_info.callable_line_entry.GetFile().GetFilename().GetCString(), callable_info.callable_line_entry.line); break; case CPPLanguageRuntime::LibCppStdFunctionCallableCase::CallableObject: stream.Printf( " Function in File %s at Line %u", - callable_info.callable_line_entry.file.GetFilename().GetCString(), + callable_info.callable_line_entry.GetFile().GetFilename().GetCString(), callable_info.callable_line_entry.line); break; case CPPLanguageRuntime::LibCppStdFunctionCallableCase::FreeOrMemberFunction: diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp index 1b3cd23d9400..ddeacf18e855 100644 --- a/lldb/source/Symbol/CompileUnit.cpp +++ b/lldb/source/Symbol/CompileUnit.cpp @@ -320,7 +320,7 @@ void CompileUnit::ResolveSymbolContext( src_location_spec.GetColumn() ? std::optional(line_entry.column) : std::nullopt; - SourceLocationSpec found_entry(line_entry.file, line_entry.line, column, + SourceLocationSpec found_entry(line_entry.GetFile(), line_entry.line, column, inlines, exact); while (line_idx != UINT32_MAX) { diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp index fdc090355771..194f89bc51d8 100644 --- a/lldb/source/Symbol/Function.cpp +++ b/lldb/source/Symbol/Function.cpp @@ -289,7 +289,7 @@ void Function::GetStartLineSourceInfo(FileSpec &source_file, if (line_table->FindLineEntryByAddress(GetAddressRange().GetBaseAddress(), line_entry, nullptr)) { line_no = line_entry.line; - source_file = line_entry.file; + source_file = line_entry.GetFile(); } } } @@ -311,7 +311,7 @@ void Function::GetEndLineSourceInfo(FileSpec &source_file, uint32_t &line_no) { LineEntry line_entry; if (line_table->FindLineEntryByAddress(scratch_addr, line_entry, nullptr)) { line_no = line_entry.line; - source_file = line_entry.file; + source_file = line_entry.GetFile(); } } diff --git a/lldb/source/Symbol/LineEntry.cpp b/lldb/source/Symbol/LineEntry.cpp index 389f8dcb65d8..9e0c06b6ff73 100644 --- a/lldb/source/Symbol/LineEntry.cpp +++ b/lldb/source/Symbol/LineEntry.cpp @@ -14,12 +14,12 @@ using namespace lldb_private; LineEntry::LineEntry() - : range(), file(), is_start_of_statement(0), is_start_of_basic_block(0), + : range(), is_start_of_statement(0), is_start_of_basic_block(0), is_prologue_end(0), is_epilogue_begin(0), is_terminal_entry(0) {} void LineEntry::Clear() { range.Clear(); - file.Clear(); + file_sp = std::make_shared(); original_file_sp = std::make_shared(); line = LLDB_INVALID_LINE_NUMBER; column = 0; @@ -35,6 +35,7 @@ bool LineEntry::IsValid() const { } bool LineEntry::DumpStopContext(Stream *s, bool show_fullpaths) const { + const FileSpec &file = file_sp->GetSpecOnly(); if (file) { if (show_fullpaths) file.Dump(s->AsRawOstream()); @@ -67,7 +68,7 @@ bool LineEntry::Dump(Stream *s, Target *target, bool show_file, return false; } if (show_file) - *s << ", file = " << file; + *s << ", file = " << GetFile(); if (line) s->Printf(", line = %u", line); if (column) @@ -103,7 +104,7 @@ bool LineEntry::GetDescription(Stream *s, lldb::DescriptionLevel level, Address::DumpStyleFileAddress); } - *s << ": " << file; + *s << ": " << GetFile(); if (line) { s->Printf(":%u", line); @@ -173,7 +174,7 @@ int LineEntry::Compare(const LineEntry &a, const LineEntry &b) { if (a.column > b.column) return +1; - return FileSpec::Compare(a.file, b.file, true); + return FileSpec::Compare(a.GetFile(), b.GetFile(), true); } AddressRange LineEntry::GetSameLineContiguousAddressRange( @@ -242,6 +243,6 @@ void LineEntry::ApplyFileMappings(lldb::TargetSP target_sp) { // Apply any file remappings to our file. if (auto new_file_spec = target_sp->GetSourcePathMap().FindFile( original_file_sp->GetSpecOnly())) - file = *new_file_spec; + file_sp->Update(*new_file_spec); } } diff --git a/lldb/source/Symbol/LineTable.cpp b/lldb/source/Symbol/LineTable.cpp index 444135f63bc0..06cf4f698316 100644 --- a/lldb/source/Symbol/LineTable.cpp +++ b/lldb/source/Symbol/LineTable.cpp @@ -288,8 +288,8 @@ bool LineTable::ConvertEntryAtIndexToLineEntry(uint32_t idx, else line_entry.range.SetByteSize(0); - line_entry.file = - m_comp_unit->GetSupportFiles().GetFileSpecAtIndex(entry.file_idx); + line_entry.file_sp = std::make_shared( + m_comp_unit->GetSupportFiles().GetFileSpecAtIndex(entry.file_idx)); line_entry.original_file_sp = m_comp_unit->GetSupportFiles().GetSupportFileAtIndex(entry.file_idx); line_entry.line = entry.line; diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp index 3c70b8d8743c..f368896fbad4 100644 --- a/lldb/source/Symbol/SymbolContext.cpp +++ b/lldb/source/Symbol/SymbolContext.cpp @@ -472,8 +472,8 @@ bool SymbolContext::GetParentOfInlinedScope(const Address &curr_frame_pc, curr_inlined_block->GetInlinedFunctionInfo(); next_frame_pc = range.GetBaseAddress(); next_frame_sc.line_entry.range.GetBaseAddress() = next_frame_pc; - next_frame_sc.line_entry.file = - curr_inlined_block_inlined_info->GetCallSite().GetFile(); + next_frame_sc.line_entry.file_sp = std::make_shared( + curr_inlined_block_inlined_info->GetCallSite().GetFile()); next_frame_sc.line_entry.original_file_sp = std::make_shared( curr_inlined_block_inlined_info->GetCallSite().GetFile()); diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp index c29a71d92572..3af62f52d575 100644 --- a/lldb/source/Target/StackFrame.cpp +++ b/lldb/source/Target/StackFrame.cpp @@ -1922,7 +1922,7 @@ bool StackFrame::GetStatus(Stream &strm, bool show_frame_info, bool show_source, size_t num_lines = target->GetSourceManager().DisplaySourceLinesWithLineNumbers( - m_sc.line_entry.file, start_line, m_sc.line_entry.column, + m_sc.line_entry.GetFile(), start_line, m_sc.line_entry.column, source_lines_before, source_lines_after, "->", &strm); if (num_lines != 0) have_source = true; diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp index 2273e52e2e04..314b5e39c716 100644 --- a/lldb/source/Target/StackFrameList.cpp +++ b/lldb/source/Target/StackFrameList.cpp @@ -884,9 +884,9 @@ void StackFrameList::SetDefaultFileAndLineToSelectedFrame() { GetFrameAtIndex(GetSelectedFrameIndex(DoNoSelectMostRelevantFrame))); if (frame_sp) { SymbolContext sc = frame_sp->GetSymbolContext(eSymbolContextLineEntry); - if (sc.line_entry.file) + if (sc.line_entry.GetFile()) m_thread.CalculateTarget()->GetSourceManager().SetDefaultFileAndLine( - sc.line_entry.file, sc.line_entry.line); + sc.line_entry.GetFile(), sc.line_entry.line); } } } diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp index 4dfad23b56e2..412e44ede9c1 100644 --- a/lldb/source/Target/Thread.cpp +++ b/lldb/source/Target/Thread.cpp @@ -302,10 +302,10 @@ bool Thread::SetSelectedFrameByIndexNoisily(uint32_t frame_idx, SymbolContext frame_sc( frame_sp->GetSymbolContext(eSymbolContextLineEntry)); const Debugger &debugger = GetProcess()->GetTarget().GetDebugger(); - if (debugger.GetUseExternalEditor() && frame_sc.line_entry.file && + if (debugger.GetUseExternalEditor() && frame_sc.line_entry.GetFile() && frame_sc.line_entry.line != 0) { if (llvm::Error e = Host::OpenFileInExternalEditor( - debugger.GetExternalEditor(), frame_sc.line_entry.file, + debugger.GetExternalEditor(), frame_sc.line_entry.GetFile(), frame_sc.line_entry.line)) { LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e), "OpenFileInExternalEditor failed: {0}"); @@ -1753,10 +1753,10 @@ size_t Thread::GetStatus(Stream &strm, uint32_t start_frame, if (frame_sp) { SymbolContext frame_sc( frame_sp->GetSymbolContext(eSymbolContextLineEntry)); - if (frame_sc.line_entry.line != 0 && frame_sc.line_entry.file) { + if (frame_sc.line_entry.line != 0 && frame_sc.line_entry.GetFile()) { if (llvm::Error e = Host::OpenFileInExternalEditor( target->GetDebugger().GetExternalEditor(), - frame_sc.line_entry.file, frame_sc.line_entry.line)) { + frame_sc.line_entry.GetFile(), frame_sc.line_entry.line)) { LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e), "OpenFileInExternalEditor failed: {0}"); } diff --git a/lldb/source/Target/TraceDumper.cpp b/lldb/source/Target/TraceDumper.cpp index e92419e70b32..4ef8efc1a676 100644 --- a/lldb/source/Target/TraceDumper.cpp +++ b/lldb/source/Target/TraceDumper.cpp @@ -57,7 +57,7 @@ static bool FileLineAndColumnMatches(const LineEntry &a, const LineEntry &b) { return false; if (a.column != b.column) return false; - return a.file == b.file; + return a.GetFile() == b.GetFile(); } /// Compare the symbol contexts of the provided \a SymbolInfo @@ -396,7 +396,7 @@ public: m_j.attribute( "source", ToOptionalString( - item.symbol_info->sc.line_entry.file.GetPath().c_str())); + item.symbol_info->sc.line_entry.GetFile().GetPath().c_str())); m_j.attribute("line", item.symbol_info->sc.line_entry.line); m_j.attribute("column", item.symbol_info->sc.line_entry.column); } diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp index f237dd63ab1c..4379ffac9d74 100644 --- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp +++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp @@ -102,7 +102,7 @@ protected: EXPECT_EQ(line, entry.line); EXPECT_EQ(address, entry.range.GetBaseAddress()); - EXPECT_TRUE(FileSpecMatchesAsBaseOrFull(spec, entry.file)); + EXPECT_TRUE(FileSpecMatchesAsBaseOrFull(spec, entry.GetFile())); } bool ContainsCompileUnit(const SymbolContextList &sc_list, -- GitLab From 6295e677220bb6ec1fa8abe2f4a94b513b91b786 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 17:05:17 -0700 Subject: [PATCH 066/933] [Float2Int] Pre-commit test for SIToFP/UIToFP ConstantRange bug. NFC The range for these operations is being constructed without the maximum value for the range due to an incorrect usage of the ConstantRange constructor. This causes Float2Int to think the range for 'uitofp i1' only contains 0 instead of 0 and 1. --- llvm/test/Transforms/Float2Int/pr79158.ll | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 llvm/test/Transforms/Float2Int/pr79158.ll diff --git a/llvm/test/Transforms/Float2Int/pr79158.ll b/llvm/test/Transforms/Float2Int/pr79158.ll new file mode 100644 index 000000000000..d041e01a4b59 --- /dev/null +++ b/llvm/test/Transforms/Float2Int/pr79158.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=float2int -S | FileCheck %s + +define i32 @pr79158(i32 %x) { +; CHECK-LABEL: define i32 @pr79158( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i32 +; CHECK-NEXT: [[MUL1:%.*]] = mul i32 [[TMP0]], 2147483647 +; CHECK-NEXT: ret i32 [[MUL1]] +; +entry: + %cmp = icmp sgt i32 %x, 0 + %conv = uitofp i1 %cmp to double + %mul = fmul double %conv, 0x41EFFFFFFFE00000 + %conv1 = fptoui double %mul to i32 + ret i32 %conv1 +} -- GitLab From 38f8a3cf0d75cd25e13d3757027f7356e4466cb9 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:49:27 -0700 Subject: [PATCH 067/933] [mlir][spirv] Improve folding of MemRef to SPIRV Lowering (#85433) Investigate the lowering of MemRef Load/Store ops and implement additional folding of created ops Aims to improve readability of generated lowered SPIR-V code. Part of work llvm#70704 --- .../MemRefToSPIRV/MemRefToSPIRV.cpp | 52 +++--- .../SPIRV/Transforms/SPIRVConversion.cpp | 9 +- .../Conversion/GPUToSPIRV/load-store.mlir | 8 +- .../MemRefToSPIRV/bitwidth-emulation.mlir | 158 +++++------------- .../MemRefToSPIRV/memref-to-spirv.mlir | 34 +--- mlir/test/Conversion/SCFToSPIRV/for.mlir | 12 +- .../TensorToSPIRV/tensor-ops-to-spirv.mlir | 10 +- .../VectorToSPIRV/vector-to-spirv.mlir | 20 +-- 8 files changed, 93 insertions(+), 210 deletions(-) diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp index 0acb2142f3f6..81b9f55cac80 100644 --- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp +++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp @@ -50,11 +50,12 @@ static Value getOffsetForBitwidth(Location loc, Value srcIdx, int sourceBits, assert(targetBits % sourceBits == 0); Type type = srcIdx.getType(); IntegerAttr idxAttr = builder.getIntegerAttr(type, targetBits / sourceBits); - auto idx = builder.create(loc, type, idxAttr); + auto idx = builder.createOrFold(loc, type, idxAttr); IntegerAttr srcBitsAttr = builder.getIntegerAttr(type, sourceBits); - auto srcBitsValue = builder.create(loc, type, srcBitsAttr); - auto m = builder.create(loc, srcIdx, idx); - return builder.create(loc, type, m, srcBitsValue); + auto srcBitsValue = + builder.createOrFold(loc, type, srcBitsAttr); + auto m = builder.createOrFold(loc, srcIdx, idx); + return builder.createOrFold(loc, type, m, srcBitsValue); } /// Returns an adjusted spirv::AccessChainOp. Based on the @@ -74,11 +75,11 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter, Value lastDim = op->getOperand(op.getNumOperands() - 1); Type type = lastDim.getType(); IntegerAttr attr = builder.getIntegerAttr(type, targetBits / sourceBits); - auto idx = builder.create(loc, type, attr); + auto idx = builder.createOrFold(loc, type, attr); auto indices = llvm::to_vector<4>(op.getIndices()); // There are two elements if this is a 1-D tensor. assert(indices.size() == 2); - indices.back() = builder.create(loc, lastDim, idx); + indices.back() = builder.createOrFold(loc, lastDim, idx); Type t = typeConverter.convertType(op.getComponentPtr().getType()); return builder.create(loc, t, op.getBasePtr(), indices); } @@ -91,7 +92,8 @@ static Value castBoolToIntN(Location loc, Value srcBool, Type dstType, return srcBool; Value zero = spirv::ConstantOp::getZero(dstType, loc, builder); Value one = spirv::ConstantOp::getOne(dstType, loc, builder); - return builder.create(loc, dstType, srcBool, one, zero); + return builder.createOrFold(loc, dstType, srcBool, one, + zero); } /// Returns the `targetBits`-bit value shifted by the given `offset`, and cast @@ -111,10 +113,10 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask, loc, builder.getIntegerType(targetBits), value); } - value = builder.create(loc, value, mask); + value = builder.createOrFold(loc, value, mask); } - return builder.create(loc, value.getType(), value, - offset); + return builder.createOrFold(loc, value.getType(), + value, offset); } /// Returns true if the allocations of memref `type` generated from `allocOp` @@ -165,7 +167,7 @@ static Value castIntNToBool(Location loc, Value srcInt, OpBuilder &builder) { return srcInt; auto one = spirv::ConstantOp::getOne(srcInt.getType(), loc, builder); - return builder.create(loc, srcInt, one); + return builder.createOrFold(loc, srcInt, one); } //===----------------------------------------------------------------------===// @@ -597,13 +599,14 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, // ____XXXX________ -> ____________XXXX Value lastDim = accessChainOp->getOperand(accessChainOp.getNumOperands() - 1); Value offset = getOffsetForBitwidth(loc, lastDim, srcBits, dstBits, rewriter); - Value result = rewriter.create( + Value result = rewriter.createOrFold( loc, spvLoadOp.getType(), spvLoadOp, offset); // Apply the mask to extract corresponding bits. - Value mask = rewriter.create( + Value mask = rewriter.createOrFold( loc, dstType, rewriter.getIntegerAttr(dstType, (1 << srcBits) - 1)); - result = rewriter.create(loc, dstType, result, mask); + result = + rewriter.createOrFold(loc, dstType, result, mask); // Apply sign extension on the loading value unconditionally. The signedness // semantic is carried in the operator itself, we relies other pattern to @@ -611,11 +614,11 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, IntegerAttr shiftValueAttr = rewriter.getIntegerAttr(dstType, dstBits - srcBits); Value shiftValue = - rewriter.create(loc, dstType, shiftValueAttr); - result = rewriter.create(loc, dstType, result, - shiftValue); - result = rewriter.create(loc, dstType, result, - shiftValue); + rewriter.createOrFold(loc, dstType, shiftValueAttr); + result = rewriter.createOrFold(loc, dstType, + result, shiftValue); + result = rewriter.createOrFold( + loc, dstType, result, shiftValue); rewriter.replaceOp(loadOp, result); @@ -744,11 +747,12 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, // Create a mask to clear the destination. E.g., if it is the second i8 in // i32, 0xFFFF00FF is created. - Value mask = rewriter.create( + Value mask = rewriter.createOrFold( loc, dstType, rewriter.getIntegerAttr(dstType, (1 << srcBits) - 1)); - Value clearBitsMask = - rewriter.create(loc, dstType, mask, offset); - clearBitsMask = rewriter.create(loc, dstType, clearBitsMask); + Value clearBitsMask = rewriter.createOrFold( + loc, dstType, mask, offset); + clearBitsMask = + rewriter.createOrFold(loc, dstType, clearBitsMask); Value storeVal = shiftValue(loc, adaptor.getValue(), offset, mask, rewriter); Value adjustedPtr = adjustAccessChainForBitwidth(typeConverter, accessChainOp, @@ -910,7 +914,7 @@ LogicalResult ReinterpretCastPattern::matchAndRewrite( int64_t attrVal = cast(offset.get()).getInt(); Attribute attr = rewriter.getIntegerAttr(intType, attrVal); - return rewriter.create(loc, intType, attr); + return rewriter.createOrFold(loc, intType, attr); }(); rewriter.replaceOpWithNewOp( diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp index 2b79c8022b8e..4072608dc8f8 100644 --- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp +++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp @@ -991,15 +991,16 @@ Value mlir::spirv::linearizeIndex(ValueRange indices, ArrayRef strides, // broken down into progressive small steps so we can have intermediate steps // using other dialects. At the moment SPIR-V is the final sink. - Value linearizedIndex = builder.create( + Value linearizedIndex = builder.createOrFold( loc, integerType, IntegerAttr::get(integerType, offset)); for (const auto &index : llvm::enumerate(indices)) { - Value strideVal = builder.create( + Value strideVal = builder.createOrFold( loc, integerType, IntegerAttr::get(integerType, strides[index.index()])); - Value update = builder.create(loc, strideVal, index.value()); + Value update = + builder.createOrFold(loc, index.value(), strideVal); linearizedIndex = - builder.create(loc, linearizedIndex, update); + builder.createOrFold(loc, update, linearizedIndex); } return linearizedIndex; } diff --git a/mlir/test/Conversion/GPUToSPIRV/load-store.mlir b/mlir/test/Conversion/GPUToSPIRV/load-store.mlir index fa12da8ef9d4..4339799ccd5e 100644 --- a/mlir/test/Conversion/GPUToSPIRV/load-store.mlir +++ b/mlir/test/Conversion/GPUToSPIRV/load-store.mlir @@ -60,13 +60,9 @@ module attributes { // CHECK: %[[INDEX2:.*]] = spirv.IAdd %[[ARG4]], %[[LOCALINVOCATIONIDX]] %13 = arith.addi %arg4, %3 : index // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 - // CHECK: %[[OFFSET1_0:.*]] = spirv.Constant 0 : i32 // CHECK: %[[STRIDE1_1:.*]] = spirv.Constant 4 : i32 - // CHECK: %[[UPDATE1_1:.*]] = spirv.IMul %[[STRIDE1_1]], %[[INDEX1]] : i32 - // CHECK: %[[OFFSET1_1:.*]] = spirv.IAdd %[[OFFSET1_0]], %[[UPDATE1_1]] : i32 - // CHECK: %[[STRIDE1_2:.*]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE1_2:.*]] = spirv.IMul %[[STRIDE1_2]], %[[INDEX2]] : i32 - // CHECK: %[[OFFSET1_2:.*]] = spirv.IAdd %[[OFFSET1_1]], %[[UPDATE1_2]] : i32 + // CHECK: %[[UPDATE1_1:.*]] = spirv.IMul %[[INDEX1]], %[[STRIDE1_1]] : i32 + // CHECK: %[[OFFSET1_2:.*]] = spirv.IAdd %[[INDEX2]], %[[UPDATE1_1]] : i32 // CHECK: %[[PTR1:.*]] = spirv.AccessChain %[[ARG0]]{{\[}}%[[ZERO]], %[[OFFSET1_2]]{{\]}} // CHECK-NEXT: %[[VAL1:.*]] = spirv.Load "StorageBuffer" %[[PTR1]] %14 = memref.load %arg0[%12, %13] : memref<12x4xf32, #spirv.storage_class> diff --git a/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir b/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir index 470c8531e2e0..52ed14e8cce2 100644 --- a/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir +++ b/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir @@ -12,16 +12,10 @@ module attributes { // CHECK-LABEL: @load_i1 func.func @load_i1(%arg0: memref>) -> i1 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[T4:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -37,32 +31,20 @@ func.func @load_i1(%arg0: memref>) -> i1 // INDEX64-LABEL: @load_i8 func.func @load_i8(%arg0: memref>) -> i8 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 // CHECK: builtin.unrealized_conversion_cast %[[SR]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] : {{.+}}, i64, i64 + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 // INDEX64: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i64 // INDEX64: %[[MASK:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // INDEX64: %[[T2:.+]] = spirv.Constant 24 : i32 // INDEX64: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // INDEX64: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -76,15 +58,12 @@ func.func @load_i8(%arg0: memref>) -> i8 func.func @load_i16(%arg0: memref<10xi16, #spirv.storage_class>, %index : index) -> i16 { // CHECK: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE:.+]] = spirv.IMul %[[ONE]], %[[ARG1_CAST]] : i32 - // CHECK: %[[FLAT_IDX:.+]] = spirv.IAdd %[[ZERO]], %[[UPDATE]] : i32 // CHECK: %[[TWO:.+]] = spirv.Constant 2 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] // CHECK: %[[SIXTEEN:.+]] = spirv.Constant 16 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[SIXTEEN]] : i32 // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 65535 : i32 @@ -110,20 +89,12 @@ func.func @load_f32(%arg0: memref>) { func.func @store_i1(%arg0: memref>, %value: i1) { // CHECK: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK:.+]] = spirv.Constant -256 : i32 // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 // CHECK: %[[CASTED_ARG1:.+]] = spirv.Select %[[ARG1]], %[[ONE]], %[[ZERO]] : i1, i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CASTED_ARG1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[CASTED_ARG1]] memref.store %value, %arg0[] : memref> return } @@ -136,36 +107,22 @@ func.func @store_i8(%arg0: memref>, %val // CHECK-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : i8 to i32 // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK2:.+]] = spirv.Constant -256 : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] - // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] + // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK2]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[CLAMPED_VAL]] // INDEX64-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : i8 to i32 // INDEX64-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 // INDEX64: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // INDEX64: %[[MASK2:.+]] = spirv.Constant -256 : i32 // INDEX64: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // INDEX64: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] : {{.+}}, i64, i64 - // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // INDEX64: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 + // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK2]] + // INDEX64: spirv.AtomicOr %[[PTR]], %[[CLAMPED_VAL]] memref.store %value, %arg0[] : memref> return } @@ -177,19 +134,16 @@ func.func @store_i16(%arg0: memref<10xi16, #spirv.storage_class>, // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE:.+]] = spirv.IMul %[[ONE]], %[[ARG1_CAST]] : i32 - // CHECK: %[[FLAT_IDX:.+]] = spirv.IAdd %[[ZERO]], %[[UPDATE]] : i32 // CHECK: %[[TWO:.+]] = spirv.Constant 2 : i32 // CHECK: %[[SIXTEEN:.+]] = spirv.Constant 16 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[SIXTEEN]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 65535 : i32 // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG2_CAST]], %[[MASK1]] : i32 // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] @@ -222,15 +176,12 @@ module attributes { func.func @load_i4(%arg0: memref>, %i: index) -> i4 { // CHECK: %[[INDEX:.+]] = builtin.unrealized_conversion_cast %{{.+}} : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[INDEX]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[FOUR]] : i32 // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 15 : i32 @@ -248,19 +199,16 @@ func.func @store_i4(%arg0: memref>, %v // CHECK: %[[VAL:.+]] = builtin.unrealized_conversion_cast %{{.+}} : i4 to i32 // CHECK: %[[INDEX:.+]] = builtin.unrealized_conversion_cast %{{.+}} : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[INDEX]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant [[OFFSET]] : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[FOUR]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 15 : i32 // CHECK: %[[SL:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[BITS]] : i32, i32 // CHECK: %[[MASK2:.+]] = spirv.Not %[[SL]] : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[VAL]], %[[MASK1]] : i32 // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[BITS]] : i32, i32 - // CHECK: %[[ACCESS_INDEX:.+]] = spirv.SDiv %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[ACCESS_INDEX:.+]] = spirv.SDiv %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ACCESS_INDEX]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK2]] // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] @@ -283,16 +231,10 @@ module attributes { // INDEX64-LABEL: @load_i8 func.func @load_i8(%arg0: memref>) -> i8 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -300,16 +242,10 @@ func.func @load_i8(%arg0: memref>) -> i8 // CHECK: return %[[CAST]] : i8 // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] : {{.+}}, i64, i64 + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 // INDEX64: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i64 // INDEX64: %[[MASK:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // INDEX64: %[[T2:.+]] = spirv.Constant 24 : i32 // INDEX64: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // INDEX64: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -326,37 +262,19 @@ func.func @load_i8(%arg0: memref>) -> i8 func.func @store_i8(%arg0: memref>, %value: i8) { // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK1:.+]] = spirv.Constant -256 : i32 // CHECK: %[[ARG1_CAST:.+]] = spirv.UConvert %[[ARG1]] : i8 to i32 - // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] - // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] + // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK1]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[ARG1_CAST]] // INDEX64-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // INDEX64: %[[MASK1:.+]] = spirv.Constant -256 : i32 // INDEX64: %[[ARG1_CAST:.+]] = spirv.UConvert %[[ARG1]] : i8 to i32 - // INDEX64: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // INDEX64: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] : {{.+}}, i64, i64 - // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // INDEX64: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 + // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK1]] + // INDEX64: spirv.AtomicOr %[[PTR]], %[[ARG1_CAST]] memref.store %value, %arg0[] : memref> return } diff --git a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir index feb6d4e92401..10c03a270005 100644 --- a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir +++ b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir @@ -70,11 +70,8 @@ func.func @load_store_unknown_dim(%i: index, %source: memref>, %i : index) -> i1 { // CHECK-DAG: %[[SRC_CAST:.+]] = builtin.unrealized_conversion_cast %[[SRC]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr [0])>, StorageBuffer> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ZERO]], %[[ADD]]] + // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ZERO]], %[[IDX_CAST]]] // CHECK: %[[VAL:.+]] = spirv.Load "StorageBuffer" %[[ADDR]] : i8 // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 // CHECK: %[[BOOL:.+]] = spirv.IEqual %[[VAL]], %[[ONE_I8]] : i8 @@ -90,15 +87,10 @@ func.func @store_i1(%dst: memref<4xi1, #spirv.storage_class>, %i: %true = arith.constant true // CHECK-DAG: %[[DST_CAST:.+]] = builtin.unrealized_conversion_cast %[[DST]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr [0])>, StorageBuffer> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ZERO]], %[[ADD]]] - // CHECK: %[[ZERO_I8:.+]] = spirv.Constant 0 : i8 + // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ZERO]], %[[IDX_CAST]]] // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 - // CHECK: %[[RES:.+]] = spirv.Select %{{.+}}, %[[ONE_I8]], %[[ZERO_I8]] : i1, i8 - // CHECK: spirv.Store "StorageBuffer" %[[ADDR]], %[[RES]] : i8 + // CHECK: spirv.Store "StorageBuffer" %[[ADDR]], %[[ONE_I8]] : i8 memref.store %true, %dst[%i]: memref<4xi1, #spirv.storage_class> return } @@ -234,11 +226,7 @@ func.func @load_store_unknown_dim(%i: index, %source: memref>, %i : index) -> i1 { // CHECK-DAG: %[[SRC_CAST:.+]] = builtin.unrealized_conversion_cast %[[SRC]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr, CrossWorkgroup> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ADD]]] + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[IDX_CAST]]] // CHECK: %[[VAL:.+]] = spirv.Load "CrossWorkgroup" %[[ADDR]] : i8 // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 // CHECK: %[[BOOL:.+]] = spirv.IEqual %[[VAL]], %[[ONE_I8]] : i8 @@ -254,15 +242,9 @@ func.func @store_i1(%dst: memref<4xi1, #spirv.storage_class>, %i %true = arith.constant true // CHECK-DAG: %[[DST_CAST:.+]] = builtin.unrealized_conversion_cast %[[DST]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr, CrossWorkgroup> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ADD]]] - // CHECK: %[[ZERO_I8:.+]] = spirv.Constant 0 : i8 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[IDX_CAST]]] // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 - // CHECK: %[[RES:.+]] = spirv.Select %{{.+}}, %[[ONE_I8]], %[[ZERO_I8]] : i1, i8 - // CHECK: spirv.Store "CrossWorkgroup" %[[ADDR]], %[[RES]] : i8 + // CHECK: spirv.Store "CrossWorkgroup" %[[ADDR]], %[[ONE_I8]] : i8 memref.store %true, %dst[%i]: memref<4xi1, #spirv.storage_class> return } diff --git a/mlir/test/Conversion/SCFToSPIRV/for.mlir b/mlir/test/Conversion/SCFToSPIRV/for.mlir index 02558463b866..81661ec7a3a0 100644 --- a/mlir/test/Conversion/SCFToSPIRV/for.mlir +++ b/mlir/test/Conversion/SCFToSPIRV/for.mlir @@ -19,17 +19,9 @@ func.func @loop_kernel(%arg2 : memref<10xf32, #spirv.storage_class i32 { // CHECK: spirv.Store "Function" %[[VAR]], %[[CST]] : !spirv.array<12 x i32> // CHECK: %[[C0:.+]] = spirv.Constant 0 : i32 // CHECK: %[[C6:.+]] = spirv.Constant 6 : i32 - // CHECK: %[[MUL0:.+]] = spirv.IMul %[[C6]], %[[A]] : i32 - // CHECK: %[[ADD0:.+]] = spirv.IAdd %[[C0]], %[[MUL0]] : i32 + // CHECK: %[[MUL0:.+]] = spirv.IMul %[[A]], %[[C6]] : i32 // CHECK: %[[C3:.+]] = spirv.Constant 3 : i32 - // CHECK: %[[MUL1:.+]] = spirv.IMul %[[C3]], %[[B]] : i32 - // CHECK: %[[ADD1:.+]] = spirv.IAdd %[[ADD0]], %[[MUL1]] : i32 + // CHECK: %[[MUL1:.+]] = spirv.IMul %[[B]], %[[C3]] : i32 + // CHECK: %[[ADD1:.+]] = spirv.IAdd %[[MUL1]], %[[MUL0]] : i32 // CHECK: %[[C1:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL2:.+]] = spirv.IMul %[[C1]], %[[C]] : i32 - // CHECK: %[[ADD2:.+]] = spirv.IAdd %[[ADD1]], %[[MUL2]] : i32 + // CHECK: %[[ADD2:.+]] = spirv.IAdd %[[C]], %[[ADD1]] : i32 // CHECK: %[[AC:.+]] = spirv.AccessChain %[[VAR]][%[[ADD2]]] // CHECK: %[[VAL:.+]] = spirv.Load "Function" %[[AC]] : i32 %extract = tensor.extract %cst[%a, %b, %c] : tensor<2x2x3xi32> diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir index c9984091d5ac..cddc4ee38535 100644 --- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir +++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir @@ -720,9 +720,7 @@ module attributes { // CHECK: %[[CST1:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST2:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST3:.+]] = spirv.Constant 1 : i32 -// CHECK: %[[S2:.+]] = spirv.IMul %[[CST3]], %[[S1]] : i32 -// CHECK: %[[S3:.+]] = spirv.IAdd %[[CST2]], %[[S2]] : i32 -// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S3]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 +// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S1]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S5:.+]] = spirv.Bitcast %[[S4]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: %[[R0:.+]] = spirv.Load "StorageBuffer" %[[S5]] : vector<4xf32> // CHECK: return %[[R0]] : vector<4xf32> @@ -743,11 +741,9 @@ func.func @vector_load(%arg0 : memref<4xf32, #spirv.storage_class // CHECK: %[[CST0_1:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST0_2:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST4:.+]] = spirv.Constant 4 : i32 -// CHECK: %[[S3:.+]] = spirv.IMul %[[CST4]], %[[S1]] : i32 -// CHECK: %[[S4:.+]] = spirv.IAdd %[[CST0_2]], %[[S3]] : i32 +// CHECK: %[[S3:.+]] = spirv.IMul %[[S1]], %[[CST4]] : i32 // CHECK: %[[CST1:.+]] = spirv.Constant 1 : i32 -// CHECK: %[[S5:.+]] = spirv.IMul %[[CST1]], %[[S2]] : i32 -// CHECK: %[[S6:.+]] = spirv.IAdd %[[S4]], %[[S5]] : i32 +// CHECK: %[[S6:.+]] = spirv.IAdd %[[S2]], %[[S3]] : i32 // CHECK: %[[S7:.+]] = spirv.AccessChain %[[S0]][%[[CST0_1]], %[[S6]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S8:.+]] = spirv.Bitcast %[[S7]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: %[[R0:.+]] = spirv.Load "StorageBuffer" %[[S8]] : vector<4xf32> @@ -768,9 +764,7 @@ func.func @vector_load_2d(%arg0 : memref<4x4xf32, #spirv.storage_class [0])>, StorageBuffer>, i32, i32 +// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S1]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S5:.+]] = spirv.Bitcast %[[S4]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: spirv.Store "StorageBuffer" %[[S5]], %[[ARG1]] : vector<4xf32> func.func @vector_store(%arg0 : memref<4xf32, #spirv.storage_class>, %arg1 : vector<4xf32>) { @@ -790,11 +784,9 @@ func.func @vector_store(%arg0 : memref<4xf32, #spirv.storage_class [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S8:.+]] = spirv.Bitcast %[[S7]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: spirv.Store "StorageBuffer" %[[S8]], %[[ARG1]] : vector<4xf32> -- GitLab From 6317c780d81327bd06701a6aa374fc92aa3d73ad Mon Sep 17 00:00:00 2001 From: Chelsea Cassanova Date: Thu, 21 Mar 2024 08:49:43 -0700 Subject: [PATCH 068/933] [lldb][progress][NFC] Clarify Doxygen comments for `details` field (#86002) The Doxygen comments for the `details` field of a progress report currently does not specify that this field will act as the initial set of details for a progress report that gets updated with `Progress::Increment()`. This commit clarifies this. --- lldb/include/lldb/Core/Progress.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h index c38f6dd0a140..eb3af9816dc6 100644 --- a/lldb/include/lldb/Core/Progress.h +++ b/lldb/include/lldb/Core/Progress.h @@ -66,7 +66,11 @@ public: /// @param [in] title The title of this progress activity. /// /// @param [in] details Specific information about what the progress report - /// is currently working on. + /// is currently working on. Although not required, if the progress report is + /// updated with Progress::Increment() then this field will be overwritten + /// with the new set of details passed into that function, and the details + /// passed initially will act as an "item 0" for the total set of + /// items being reported on. /// /// @param [in] total The total units of work to be done if specified, if /// set to std::nullopt then an indeterminate progress indicator should be -- GitLab From f5c90f3000bc75a344bf01bd4e0401e3fb7f9453 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 08:52:51 -0700 Subject: [PATCH 069/933] [RISCV] Use BuildPairF64 and SplitF64 for bitcast i64<->f64 on rv32 regardless of Zfa. (#85982) Previously we used BuildPairF64 and SplitF64 only if Zfa was supported since they will select register file moves that are only available with Zfa. We recently changed the handling of BuildPairF64/SplitF64 for Zdinx to not go through memory so we should use that for bitcast. That leaves the D without Zfa case that does need to go through memory. Previously we let type legalization expand to loads and stores using a new stack temporary created for each bitcast. After this patch we will create the loads ands stores in the custom inserter and share the same stack slot for all. This also allows DAGCombiner to optimize when bitcast is mixed with BuildPairF64/SplitF64. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 11 ++++---- llvm/test/CodeGen/RISCV/double-convert.ll | 28 ++++--------------- .../RISCV/inline-asm-d-constraint-f.ll | 16 +---------- .../RISCV/rvv/fixed-vectors-bitcast.ll | 2 +- llvm/test/CodeGen/RISCV/spill-fill-fold.ll | 14 ++++++---- 5 files changed, 21 insertions(+), 50 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 3aa28215efc2..a3ebfb34ad7a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -559,11 +559,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, if (Subtarget.hasStdExtDOrZdinx()) { setOperationAction(FPLegalNodeTypes, MVT::f64, Legal); + if (!Subtarget.is64Bit()) + setOperationAction(ISD::BITCAST, MVT::i64, Custom); + if (Subtarget.hasStdExtZfa()) { setOperationAction(FPRndMode, MVT::f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); - if (!Subtarget.is64Bit()) - setOperationAction(ISD::BITCAST, MVT::i64, Custom); } else { if (Subtarget.is64Bit()) setOperationAction(FPRndMode, MVT::f64, Custom); @@ -6071,8 +6072,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0); return FPConv; } - if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32 && - Subtarget.hasStdExtZfa()) { + if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) { SDValue Lo, Hi; std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32); SDValue RetReg = @@ -12157,8 +12157,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0); Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); - } else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32 && - Subtarget.hasStdExtZfa()) { + } else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) { SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), Op0); SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll index 7a9439e5b322..c1429642962e 100644 --- a/llvm/test/CodeGen/RISCV/double-convert.ll +++ b/llvm/test/CodeGen/RISCV/double-convert.ll @@ -1116,13 +1116,7 @@ define i64 @fmv_x_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmv_x_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmv_x_d: @@ -1257,13 +1251,13 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind { ; RV32IFD-LABEL: fmv_d_x: ; RV32IFD: # %bb.0: ; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw a3, 4(sp) -; RV32IFD-NEXT: sw a2, 0(sp) -; RV32IFD-NEXT: sw a1, 12(sp) ; RV32IFD-NEXT: sw a0, 8(sp) -; RV32IFD-NEXT: fld fa5, 0(sp) +; RV32IFD-NEXT: sw a1, 12(sp) +; RV32IFD-NEXT: fld fa5, 8(sp) +; RV32IFD-NEXT: sw a2, 8(sp) +; RV32IFD-NEXT: sw a3, 12(sp) ; RV32IFD-NEXT: fld fa4, 8(sp) -; RV32IFD-NEXT: fadd.d fa0, fa4, fa5 +; RV32IFD-NEXT: fadd.d fa0, fa5, fa4 ; RV32IFD-NEXT: addi sp, sp, 16 ; RV32IFD-NEXT: ret ; @@ -1276,17 +1270,7 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmv_d_x: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmv_d_x: diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll index 71769a800c06..c480ba800c69 100644 --- a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll +++ b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll @@ -75,24 +75,10 @@ define double @constraint_f_double_abi_name(double %a) nounwind { define double @constraint_gpr(double %x) { ; RV32F-LABEL: constraint_gpr: ; RV32F: # %bb.0: -; RV32F-NEXT: addi sp, sp, -32 -; RV32F-NEXT: .cfi_def_cfa_offset 32 -; RV32F-NEXT: sw a0, 8(sp) -; RV32F-NEXT: sw a1, 12(sp) -; RV32F-NEXT: fld fa5, 8(sp) -; RV32F-NEXT: fsd fa5, 24(sp) -; RV32F-NEXT: lw a0, 24(sp) -; RV32F-NEXT: lw a1, 28(sp) +; RV32F-NEXT: .cfi_def_cfa_offset 0 ; RV32F-NEXT: #APP ; RV32F-NEXT: mv a0, a0 ; RV32F-NEXT: #NO_APP -; RV32F-NEXT: sw a1, 20(sp) -; RV32F-NEXT: sw a0, 16(sp) -; RV32F-NEXT: fld fa5, 16(sp) -; RV32F-NEXT: fsd fa5, 8(sp) -; RV32F-NEXT: lw a0, 8(sp) -; RV32F-NEXT: lw a1, 12(sp) -; RV32F-NEXT: addi sp, sp, 32 ; RV32F-NEXT: ret ; ; RV64F-LABEL: constraint_gpr: diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll index b7afee754f68..5252eb71c383 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll @@ -416,8 +416,8 @@ define double @bitcast_v1i64_f64(<1 x i64> %a) { ; RV32ELEN32: # %bb.0: ; RV32ELEN32-NEXT: addi sp, sp, -16 ; RV32ELEN32-NEXT: .cfi_def_cfa_offset 16 -; RV32ELEN32-NEXT: sw a1, 12(sp) ; RV32ELEN32-NEXT: sw a0, 8(sp) +; RV32ELEN32-NEXT: sw a1, 12(sp) ; RV32ELEN32-NEXT: fld fa0, 8(sp) ; RV32ELEN32-NEXT: addi sp, sp, 16 ; RV32ELEN32-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/spill-fill-fold.ll b/llvm/test/CodeGen/RISCV/spill-fill-fold.ll index a9a0cc5cf94d..8cf5f55ad5c5 100644 --- a/llvm/test/CodeGen/RISCV/spill-fill-fold.ll +++ b/llvm/test/CodeGen/RISCV/spill-fill-fold.ll @@ -290,8 +290,8 @@ define double @spill_i64_to_double(i64 %a) nounwind { ; RV32ID-NEXT: fsd fs9, 40(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs10, 32(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs11, 24(sp) # 8-byte Folded Spill -; RV32ID-NEXT: sw a1, 20(sp) ; RV32ID-NEXT: sw a0, 16(sp) +; RV32ID-NEXT: sw a1, 20(sp) ; RV32ID-NEXT: fld fa5, 16(sp) ; RV32ID-NEXT: fsd fa5, 8(sp) # 8-byte Folded Spill ; RV32ID-NEXT: #APP @@ -804,13 +804,15 @@ define double @fill_i64_to_double(i64 %a) nounwind { ; RV32ID-NEXT: fsd fs9, 40(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs10, 32(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs11, 24(sp) # 8-byte Folded Spill -; RV32ID-NEXT: sw a1, 20(sp) -; RV32ID-NEXT: sw a0, 16(sp) -; RV32ID-NEXT: fld fa5, 16(sp) -; RV32ID-NEXT: fsd fa5, 8(sp) # 8-byte Folded Spill +; RV32ID-NEXT: sw a1, 12(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw a0, 8(sp) # 4-byte Folded Spill ; RV32ID-NEXT: #APP ; RV32ID-NEXT: #NO_APP -; RV32ID-NEXT: fld fa0, 8(sp) # 8-byte Folded Reload +; RV32ID-NEXT: lw a0, 8(sp) # 4-byte Folded Reload +; RV32ID-NEXT: sw a0, 16(sp) +; RV32ID-NEXT: lw a0, 12(sp) # 4-byte Folded Reload +; RV32ID-NEXT: sw a0, 20(sp) +; RV32ID-NEXT: fld fa0, 16(sp) ; RV32ID-NEXT: lw ra, 172(sp) # 4-byte Folded Reload ; RV32ID-NEXT: lw s0, 168(sp) # 4-byte Folded Reload ; RV32ID-NEXT: lw s1, 164(sp) # 4-byte Folded Reload -- GitLab From 81bd799819f498a55e32599bce51fa98b2e73238 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 21 Mar 2024 09:06:02 -0700 Subject: [PATCH 070/933] [lldb] Add missing initialization in LineEntry ctor --- lldb/source/Symbol/LineEntry.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lldb/source/Symbol/LineEntry.cpp b/lldb/source/Symbol/LineEntry.cpp index 9e0c06b6ff73..461399e0326e 100644 --- a/lldb/source/Symbol/LineEntry.cpp +++ b/lldb/source/Symbol/LineEntry.cpp @@ -14,8 +14,10 @@ using namespace lldb_private; LineEntry::LineEntry() - : range(), is_start_of_statement(0), is_start_of_basic_block(0), - is_prologue_end(0), is_epilogue_begin(0), is_terminal_entry(0) {} + : range(), file_sp(std::make_shared()), + original_file_sp(std::make_shared()), + is_start_of_statement(0), is_start_of_basic_block(0), is_prologue_end(0), + is_epilogue_begin(0), is_terminal_entry(0) {} void LineEntry::Clear() { range.Clear(); -- GitLab From d8b0d8d6713d474cfd622e03090a7ad5206ee574 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 21:49:57 +0530 Subject: [PATCH 071/933] AMDGPU: Use defset to cleanup marking MFMA intrinsics as divergent (#85915) --- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 86 ++++++++++-------- .../Target/AMDGPU/AMDGPUSearchableTables.td | 87 +++---------------- 2 files changed, 59 insertions(+), 114 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 051e603c0819..fff03dee20a1 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2653,6 +2653,8 @@ class AMDGPUWmmaIntrinsicIU : // The OPSEL intrinsics read from and write to one half of the registers, selected by the op_sel bit. // The tied versions of the f16/bf16 wmma intrinsics tie the destination matrix registers to the input accumulator registers. // The content of the other 16-bit half is preserved from the input. + +defset list AMDGPUWMMAIntrinsicsGFX11 = { def int_amdgcn_wmma_f16_16x16x16_f16_tied : AMDGPUWmmaIntrinsicOPSEL; def int_amdgcn_wmma_bf16_16x16x16_bf16_tied : AMDGPUWmmaIntrinsicOPSEL; @@ -2668,6 +2670,7 @@ def int_amdgcn_wmma_i32_16x16x16_iu4 : AMDGPUWmmaIntrinsicIU; def int_amdgcn_wmma_bf16_16x16x16_bf16 : AMDGPUWmmaIntrinsicOPSEL; +} //===----------------------------------------------------------------------===// // GFX12 Intrinsics @@ -2687,20 +2690,6 @@ def int_amdgcn_permlanex16_var : ClangBuiltin<"__builtin_amdgcn_permlanex16_var" [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg>, ImmArg>, IntrNoCallback, IntrNoFree]>; - -// WMMA (Wave Matrix Multiply-Accumulate) intrinsics -// -// These operations perform a matrix multiplication and accumulation of -// the form: D = A * B + C . - -// A and B are <8 x fp8> or <8 x bf8>, but since fp8 and bf8 are not supported by llvm we use <2 x i32>. -def int_amdgcn_wmma_f32_16x16x16_fp8_fp8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_fp8_bf8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_bf8_fp8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_bf8_bf8 : AMDGPUWmmaIntrinsic; -// A and B are <16 x iu4>. -def int_amdgcn_wmma_i32_16x16x32_iu4 : AMDGPUWmmaIntrinsicIU; - // SWMMAC (Wave Matrix(sparse) Multiply-Accumulate) intrinsics // // These operations perform a sparse matrix multiplication and accumulation of @@ -2734,6 +2723,20 @@ class AMDGPUSWmmacIntrinsicIUIdx>, ImmArg>, ImmArg>] >; +defset list AMDGPUWMMAIntrinsicsGFX12 = { +// WMMA (Wave Matrix Multiply-Accumulate) intrinsics +// +// These operations perform a matrix multiplication and accumulation of +// the form: D = A * B + C . + +// A and B are <8 x fp8> or <8 x bf8>, but since fp8 and bf8 are not supported by llvm we use <2 x i32>. +def int_amdgcn_wmma_f32_16x16x16_fp8_fp8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_fp8_bf8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_bf8_fp8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_bf8_bf8 : AMDGPUWmmaIntrinsic; +// A and B are <16 x iu4>. +def int_amdgcn_wmma_i32_16x16x32_iu4 : AMDGPUWmmaIntrinsicIU; + def int_amdgcn_swmmac_f32_16x16x32_f16 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf16 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f16_16x16x32_f16 : AMDGPUSWmmacIntrinsicIdx; @@ -2745,6 +2748,7 @@ def int_amdgcn_swmmac_f32_16x16x32_fp8_fp8 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf8_fp8 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf8_bf8 : AMDGPUSWmmacIntrinsicIdx; +} def int_amdgcn_global_atomic_ordered_add_b64 : AMDGPUAtomicRtn; @@ -3012,6 +3016,7 @@ class AMDGPUMfmaIntrinsic : [IntrConvergent, IntrNoMem, ImmArg>, ImmArg>, ImmArg>]>; +defset list AMDGPUMFMAIntrinsics908 = { def int_amdgcn_mfma_f32_32x32x1f32 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x1f32 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_4x4x1f32 : AMDGPUMfmaIntrinsic; @@ -3032,6 +3037,7 @@ def int_amdgcn_mfma_f32_16x16x2bf16 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_32x32x4bf16 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUMfmaIntrinsic; +} //===----------------------------------------------------------------------===// // gfx90a intrinsics @@ -3043,6 +3049,7 @@ def int_amdgcn_flat_atomic_fadd : AMDGPUAtomicRtn; def int_amdgcn_flat_atomic_fmin : AMDGPUAtomicRtn; def int_amdgcn_flat_atomic_fmax : AMDGPUAtomicRtn; +defset list AMDGPUMFMAIntrinsics90A = { def int_amdgcn_mfma_f32_32x32x4bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x4bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_4x4x4bf16_1k : AMDGPUMfmaIntrinsic; @@ -3054,25 +3061,12 @@ def int_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f64_4x4x4f64 : AMDGPUMfmaIntrinsic; +} //===----------------------------------------------------------------------===// // gfx940 intrinsics // ===----------------------------------------------------------------------===// -// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. -def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; -def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; -def int_amdgcn_ds_fadd_v2bf16 : DefaultAttrsIntrinsic< - [llvm_v2i16_ty], - [LLVMQualPointerType<3>, llvm_v2i16_ty], - [IntrArgMemOnly, NoCapture>]>, - ClangBuiltin<"__builtin_amdgcn_ds_atomic_fadd_v2bf16">; - -def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUMfmaIntrinsic; - class AMDGPUMFp8MfmaIntrinsic : AMDGPUMfmaIntrinsic; @@ -3081,9 +3075,6 @@ multiclass AMDGPUMFp8MfmaIntrinsic { def NAME#"_"#kind : AMDGPUMFp8MfmaIntrinsic; } -defm int_amdgcn_mfma_f32_16x16x32 : AMDGPUMFp8MfmaIntrinsic; -defm int_amdgcn_mfma_f32_32x32x16 : AMDGPUMFp8MfmaIntrinsic; - // llvm.amdgcn.smfmac.?32.* vdst, srcA, srcB, srcC, index, cbsz, abid class AMDGPUMSmfmacIntrinsic : ClangBuiltin, @@ -3093,13 +3084,6 @@ class AMDGPUMSmfmacIntrinsic : [IntrConvergent, IntrNoMem, ImmArg>, ImmArg>]>; -def int_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUMSmfmacIntrinsic; - class AMDGPUMFp8SmfmacIntrinsic : AMDGPUMSmfmacIntrinsic; @@ -3108,8 +3092,34 @@ multiclass AMDGPUMFp8SmfmacIntrinsic { def NAME#"_"#kind : AMDGPUMFp8SmfmacIntrinsic; } +// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. +def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; +def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; +def int_amdgcn_ds_fadd_v2bf16 : DefaultAttrsIntrinsic< + [llvm_v2i16_ty], + [LLVMQualPointerType<3>, llvm_v2i16_ty], + [IntrArgMemOnly, NoCapture>]>, + ClangBuiltin<"__builtin_amdgcn_ds_atomic_fadd_v2bf16">; + +defset list AMDGPUMFMAIntrinsics940 = { +def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUMfmaIntrinsic; + +defm int_amdgcn_mfma_f32_16x16x32 : AMDGPUMFp8MfmaIntrinsic; +defm int_amdgcn_mfma_f32_32x32x16 : AMDGPUMFp8MfmaIntrinsic; + +def int_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUMSmfmacIntrinsic; + defm int_amdgcn_smfmac_f32_16x16x64 : AMDGPUMFp8SmfmacIntrinsic; defm int_amdgcn_smfmac_f32_32x32x32 : AMDGPUMFp8SmfmacIntrinsic; +} // llvm.amdgcn.cvt.f32.bf8 float vdst, int srcA, imm byte_sel [0..3] // byte_sel selects byte from srcA. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td index bb1c6b733729..8eb46a980148 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td @@ -354,82 +354,17 @@ def : SourceOfDivergence; def : SourceOfDivergence; def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics908 in +def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics90A in +def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics940 in +def : SourceOfDivergence; +foreach intr = AMDGPUWMMAIntrinsicsGFX11 in +def : SourceOfDivergence; +foreach intr = AMDGPUWMMAIntrinsicsGFX12 in +def : SourceOfDivergence; + def : SourceOfDivergence; // The dummy boolean output is divergent from the IR's perspective, -- GitLab From d9f0d9a1452ed78e943423c9fbbd63674625f7f5 Mon Sep 17 00:00:00 2001 From: Tarun Prabhu Date: Thu, 21 Mar 2024 10:22:33 -0600 Subject: [PATCH 072/933] [flang][NFC] Fix header guards Some header guards conflicted with clang. Fix a few others to follow the convention in the rest of the headers in flang. --- flang/include/flang/Common/Version.h | 6 +++--- flang/include/flang/Frontend/CodeGenOptions.h | 6 +++--- flang/include/flang/Frontend/LangOptions.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flang/include/flang/Common/Version.h b/flang/include/flang/Common/Version.h index b1bd2416a618..3257d4a4f645 100644 --- a/flang/include/flang/Common/Version.h +++ b/flang/include/flang/Common/Version.h @@ -12,8 +12,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_FLANG_COMMON_VERSION_H -#define LLVM_FLANG_COMMON_VERSION_H +#ifndef FORTRAN_COMMON_VERSION_H +#define FORTRAN_COMMON_VERSION_H #include "flang/Version.inc" #include "llvm/ADT/StringRef.h" @@ -53,4 +53,4 @@ std::string getFlangFullVersion(); std::string getFlangToolFullVersion(llvm::StringRef ToolName); } // namespace Fortran::common -#endif // LLVM_FLANG_COMMON_VERSION_H +#endif // FORTRAN_COMMON_VERSION_H diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h index 0c318e4023af..b0bbace82c04 100644 --- a/flang/include/flang/Frontend/CodeGenOptions.h +++ b/flang/include/flang/Frontend/CodeGenOptions.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_BASIC_CODEGENOPTIONS_H -#define LLVM_CLANG_BASIC_CODEGENOPTIONS_H +#ifndef FORTRAN_FRONTEND_CODEGENOPTIONS_H +#define FORTRAN_FRONTEND_CODEGENOPTIONS_H #include "llvm/Frontend/Debug/Options.h" #include "llvm/Frontend/Driver/CodeGenOptions.h" @@ -141,4 +141,4 @@ public: } // end namespace Fortran::frontend -#endif +#endif // FORTRAN_FRONTEND_CODEGENOPTIONS_H diff --git a/flang/include/flang/Frontend/LangOptions.h b/flang/include/flang/Frontend/LangOptions.h index 7adf2eec9ca3..7ab219581886 100644 --- a/flang/include/flang/Frontend/LangOptions.h +++ b/flang/include/flang/Frontend/LangOptions.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_FLANG_FRONTEND_LANGOPTIONS_H -#define LLVM_FLANG_FRONTEND_LANGOPTIONS_H +#ifndef FORTRAN_FRONTEND_LANGOPTIONS_H +#define FORTRAN_FRONTEND_LANGOPTIONS_H #include @@ -63,4 +63,4 @@ public: } // end namespace Fortran::frontend -#endif +#endif // FORTRAN_FRONTEND_LANGOPTIONS_H -- GitLab From 12836467b76c56872b4c22a6fd44bcda696ea720 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 09:25:13 -0700 Subject: [PATCH 073/933] [ConstantRange] Fix off by 1 bugs in UIToFP and SIToFP handling. (#86041) We were passing the min and max values of the range to the ConstantRange constructor, but the constructor expects the upper bound to 1 more than the max value so we need to add 1. We also need to use getNonEmpty so that passing 0, 0 to the constructor creates a full range rather than an empty range. And passing smin, smax+1 doesn't cause an assertion. I believe this fixes at least some of the reason #79158 was reverted. --- llvm/lib/IR/ConstantRange.cpp | 4 ++-- llvm/test/Transforms/Float2Int/pr79158.ll | 7 ++++--- llvm/unittests/IR/ConstantRangeTest.cpp | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 3394a1ec8dc4..59e7a9f5eb11 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -746,7 +746,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp, Min = Min.zext(ResultBitWidth); Max = Max.zext(ResultBitWidth); } - return ConstantRange(std::move(Min), std::move(Max)); + return getNonEmpty(std::move(Min), std::move(Max) + 1); } case Instruction::SIToFP: { // TODO: use input range if available @@ -757,7 +757,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp, SMin = SMin.sext(ResultBitWidth); SMax = SMax.sext(ResultBitWidth); } - return ConstantRange(std::move(SMin), std::move(SMax)); + return getNonEmpty(std::move(SMin), std::move(SMax) + 1); } case Instruction::FPTrunc: case Instruction::FPExt: diff --git a/llvm/test/Transforms/Float2Int/pr79158.ll b/llvm/test/Transforms/Float2Int/pr79158.ll index d041e01a4b59..5e78cc0bc66f 100644 --- a/llvm/test/Transforms/Float2Int/pr79158.ll +++ b/llvm/test/Transforms/Float2Int/pr79158.ll @@ -6,9 +6,10 @@ define i32 @pr79158(i32 %x) { ; CHECK-SAME: i32 [[X:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 -; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i32 -; CHECK-NEXT: [[MUL1:%.*]] = mul i32 [[TMP0]], 2147483647 -; CHECK-NEXT: ret i32 [[MUL1]] +; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i64 +; CHECK-NEXT: [[MUL1:%.*]] = mul i64 [[TMP0]], 4294967295 +; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[MUL1]] to i32 +; CHECK-NEXT: ret i32 [[TMP1]] ; entry: %cmp = icmp sgt i32 %x, 0 diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp index 34a162a5514e..8ec120d70e99 100644 --- a/llvm/unittests/IR/ConstantRangeTest.cpp +++ b/llvm/unittests/IR/ConstantRangeTest.cpp @@ -2479,6 +2479,24 @@ TEST_F(ConstantRangeTest, castOps) { ConstantRange IntToPtr = A.castOp(Instruction::IntToPtr, 64); EXPECT_EQ(64u, IntToPtr.getBitWidth()); EXPECT_TRUE(IntToPtr.isFullSet()); + + ConstantRange UIToFP = A.castOp(Instruction::UIToFP, 16); + EXPECT_EQ(16u, UIToFP.getBitWidth()); + EXPECT_TRUE(UIToFP.isFullSet()); + + ConstantRange UIToFP2 = A.castOp(Instruction::UIToFP, 64); + ConstantRange B(APInt(64, 0), APInt(64, 65536)); + EXPECT_EQ(64u, UIToFP2.getBitWidth()); + EXPECT_EQ(B, UIToFP2); + + ConstantRange SIToFP = A.castOp(Instruction::SIToFP, 16); + EXPECT_EQ(16u, SIToFP.getBitWidth()); + EXPECT_TRUE(SIToFP.isFullSet()); + + ConstantRange SIToFP2 = A.castOp(Instruction::SIToFP, 64); + ConstantRange C(APInt(64, -32768), APInt(64, 32768)); + EXPECT_EQ(64u, SIToFP2.getBitWidth()); + EXPECT_EQ(C, SIToFP2); } TEST_F(ConstantRangeTest, binaryAnd) { -- GitLab From 9a87d4d546a4382879b1beb96687acbad0ef4cc0 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 17:25:37 +0100 Subject: [PATCH 074/933] [libc] Add `is_constant_evaluated` type_traits (#86139) This will replace `__builtin_is_constant_evaluated` in math_extras.h. --- libc/src/__support/CPP/CMakeLists.txt | 5 +++-- libc/src/__support/CPP/type_traits.h | 1 + .../CPP/type_traits/is_constant_evaluated.h | 21 +++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 3 ++- 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 libc/src/__support/CPP/type_traits/is_constant_evaluated.h diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt index 6216505eae23..f76285be5219 100644 --- a/libc/src/__support/CPP/CMakeLists.txt +++ b/libc/src/__support/CPP/CMakeLists.txt @@ -103,23 +103,24 @@ add_header_library( type_traits HDRS type_traits.h - type_traits/always_false.h type_traits/add_lvalue_reference.h type_traits/add_pointer.h type_traits/add_rvalue_reference.h + type_traits/always_false.h type_traits/bool_constant.h type_traits/conditional.h type_traits/decay.h type_traits/enable_if.h type_traits/false_type.h type_traits/integral_constant.h - type_traits/invoke.h type_traits/invoke_result.h + type_traits/invoke.h type_traits/is_arithmetic.h type_traits/is_array.h type_traits/is_base_of.h type_traits/is_class.h type_traits/is_const.h + type_traits/is_constant_evaluated.h type_traits/is_convertible.h type_traits/is_destructible.h type_traits/is_enum.h diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h index 697cf79d6ccc..1494aeb905e0 100644 --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -25,6 +25,7 @@ #include "src/__support/CPP/type_traits/is_base_of.h" #include "src/__support/CPP/type_traits/is_class.h" #include "src/__support/CPP/type_traits/is_const.h" +#include "src/__support/CPP/type_traits/is_constant_evaluated.h" #include "src/__support/CPP/type_traits/is_convertible.h" #include "src/__support/CPP/type_traits/is_destructible.h" #include "src/__support/CPP/type_traits/is_enum.h" diff --git a/libc/src/__support/CPP/type_traits/is_constant_evaluated.h b/libc/src/__support/CPP/type_traits/is_constant_evaluated.h new file mode 100644 index 000000000000..93cfd07b567f --- /dev/null +++ b/libc/src/__support/CPP/type_traits/is_constant_evaluated.h @@ -0,0 +1,21 @@ +//===-- is_constant_evaluated type_traits -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H +#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H + +#include "src/__support/macros/attributes.h" + +namespace LIBC_NAMESPACE::cpp { + +LIBC_INLINE constexpr bool is_constant_evaluated() { + return __builtin_is_constant_evaluated(); +} + +} // namespace LIBC_NAMESPACE::cpp + +#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index fe4b8e2de14e..2e8d475f196e 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -339,6 +339,7 @@ libc_support_library( "src/__support/CPP/type_traits/is_base_of.h", "src/__support/CPP/type_traits/is_class.h", "src/__support/CPP/type_traits/is_const.h", + "src/__support/CPP/type_traits/is_constant_evaluated.h", "src/__support/CPP/type_traits/is_convertible.h", "src/__support/CPP/type_traits/is_destructible.h", "src/__support/CPP/type_traits/is_enum.h", @@ -743,12 +744,12 @@ libc_support_library( deps = [ ":__support_common", ":__support_cpp_bit", - ":__support_sign", ":__support_cpp_type_traits", ":__support_libc_assert", ":__support_macros_attributes", ":__support_macros_properties_types", ":__support_math_extras", + ":__support_sign", ":__support_uint128", ], ) -- GitLab From c1c2551a2876f536b5a06f48fa809aeedbc3d7ba Mon Sep 17 00:00:00 2001 From: OverMighty Date: Thu, 21 Mar 2024 16:33:16 +0000 Subject: [PATCH 075/933] [clang] Implement __builtin_{clzg,ctzg} (#83431) Fixes #83075, fixes #83076. --- clang/docs/LanguageExtensions.rst | 41 ++++ clang/include/clang/Basic/Builtins.td | 12 +- .../clang/Basic/DiagnosticSemaKinds.td | 15 +- clang/lib/CodeGen/CGBuiltin.cpp | 46 +++- clang/lib/Sema/SemaChecking.cpp | 47 ++++ clang/test/CodeGen/builtins.c | 204 ++++++++++++++++++ clang/test/CodeGen/ubsan-builtin-checks.c | 6 + clang/test/Sema/builtin-popcountg.c | 23 -- clang/test/Sema/count-builtins.c | 87 ++++++++ 9 files changed, 441 insertions(+), 40 deletions(-) delete mode 100644 clang/test/Sema/builtin-popcountg.c create mode 100644 clang/test/Sema/count-builtins.c diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 201a4c27f7dd..5711972b55e6 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -3553,6 +3553,47 @@ argument can be of any unsigned integer type. ``__builtin_popcount{,l,ll}`` builtins, with support for other integer types, such as ``unsigned __int128`` and C23 ``unsigned _BitInt(N)``. +``__builtin_clzg`` and ``__builtin_ctzg`` +----------------------------------------- + +``__builtin_clzg`` (respectively ``__builtin_ctzg``) returns the number of +leading (respectively trailing) 0 bits in the first argument. The first argument +can be of any unsigned integer type. + +If the first argument is 0 and an optional second argument of ``int`` type is +provided, then the second argument is returned. If the first argument is 0, but +only one argument is provided, then the behavior is undefined. + +**Syntax**: + +.. code-block:: c++ + + int __builtin_clzg(type x[, int fallback]) + int __builtin_ctzg(type x[, int fallback]) + +**Examples**: + +.. code-block:: c++ + + unsigned int x = 1; + int x_lz = __builtin_clzg(x); + int x_tz = __builtin_ctzg(x); + + unsigned long y = 2; + int y_lz = __builtin_clzg(y); + int y_tz = __builtin_ctzg(y); + + unsigned _BitInt(128) z = 4; + int z_lz = __builtin_clzg(z); + int z_tz = __builtin_ctzg(z); + +**Description**: + +``__builtin_clzg`` (respectively ``__builtin_ctzg``) is meant to be a +type-generic alternative to the ``__builtin_clz{,l,ll}`` (respectively +``__builtin_ctz{,l,ll}``) builtins, with support for other integer types, such +as ``unsigned __int128`` and C23 ``unsigned _BitInt(N)``. + Multiprecision Arithmetic Builtins ---------------------------------- diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 491c9d895413..21ab9bb86d1b 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -676,7 +676,11 @@ def Clz : Builtin, BitShort_Int_Long_LongLongTemplate { let Prototype = "int(unsigned T)"; } -// FIXME: Add int clzimax(uintmax_t) +def Clzg : Builtin { + let Spellings = ["__builtin_clzg"]; + let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Prototype = "int(...)"; +} def Ctz : Builtin, BitShort_Int_Long_LongLongTemplate { let Spellings = ["__builtin_ctz"]; @@ -684,7 +688,11 @@ def Ctz : Builtin, BitShort_Int_Long_LongLongTemplate { let Prototype = "int(unsigned T)"; } -// FIXME: Add int ctzimax(uintmax_t) +def Ctzg : Builtin { + let Spellings = ["__builtin_ctzg"]; + let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Prototype = "int(...)"; +} def FFS : Builtin, BitInt_Long_LongLongTemplate { let Spellings = ["__builtin_ffs"]; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 2646942a53e3..270af5d24611 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -12020,13 +12020,14 @@ def err_builtin_launder_invalid_arg : Error< "'__builtin_launder' is not allowed">; def err_builtin_invalid_arg_type: Error < - "%ordinal0 argument must be a " - "%select{vector, integer or floating point type|matrix|" - "pointer to a valid matrix element type|" - "signed integer or floating point type|vector type|" - "floating point type|" - "vector of integers|" - "type of unsigned integer}1 (was %2)">; + "%ordinal0 argument must be " + "%select{a vector, integer or floating point type|a matrix|" + "a pointer to a valid matrix element type|" + "a signed integer or floating point type|a vector type|" + "a floating point type|" + "a vector of integers|" + "an unsigned integer|" + "an 'int'}1 (was %2)">; def err_builtin_matrix_disabled: Error< "matrix types extension is disabled. Pass -fenable-matrix to enable it">; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 77cb269d43c5..e14e89088282 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3128,36 +3128,66 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ctzs: case Builtin::BI__builtin_ctz: case Builtin::BI__builtin_ctzl: - case Builtin::BI__builtin_ctzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); + case Builtin::BI__builtin_ctzll: + case Builtin::BI__builtin_ctzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); - return RValue::get(Result); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_clzs: case Builtin::BI__builtin_clz: case Builtin::BI__builtin_clzl: - case Builtin::BI__builtin_clzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); + case Builtin::BI__builtin_clzll: + case Builtin::BI__builtin_clzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); - return RValue::get(Result); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_ffs: case Builtin::BI__builtin_ffsl: diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index ef3ab16ba29b..246e3577809a 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2399,6 +2399,48 @@ static bool SemaBuiltinPopcountg(Sema &S, CallExpr *TheCall) { return false; } +/// Checks that __builtin_{clzg,ctzg} was called with a first argument, which is +/// an unsigned integer, and an optional second argument, which is promoted to +/// an 'int'. +static bool SemaBuiltinCountZeroBitsGeneric(Sema &S, CallExpr *TheCall) { + if (checkArgCountRange(S, TheCall, 1, 2)) + return true; + + ExprResult Arg0Res = S.DefaultLvalueConversion(TheCall->getArg(0)); + if (Arg0Res.isInvalid()) + return true; + + Expr *Arg0 = Arg0Res.get(); + TheCall->setArg(0, Arg0); + + QualType Arg0Ty = Arg0->getType(); + + if (!Arg0Ty->isUnsignedIntegerType()) { + S.Diag(Arg0->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 1 << /*unsigned integer ty*/ 7 << Arg0Ty; + return true; + } + + if (TheCall->getNumArgs() > 1) { + ExprResult Arg1Res = S.UsualUnaryConversions(TheCall->getArg(1)); + if (Arg1Res.isInvalid()) + return true; + + Expr *Arg1 = Arg1Res.get(); + TheCall->setArg(1, Arg1); + + QualType Arg1Ty = Arg1->getType(); + + if (!Arg1Ty->isSpecificBuiltinType(BuiltinType::Int)) { + S.Diag(Arg1->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 2 << /*'int' ty*/ 8 << Arg1Ty; + return true; + } + } + + return false; +} + ExprResult Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, CallExpr *TheCall) { @@ -3187,6 +3229,11 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, if (SemaBuiltinPopcountg(*this, TheCall)) return ExprError(); break; + case Builtin::BI__builtin_clzg: + case Builtin::BI__builtin_ctzg: + if (SemaBuiltinCountZeroBitsGeneric(*this, TheCall)) + return ExprError(); + break; } if (getLangOpts().HLSL && CheckHLSLBuiltinFunctionCall(BuiltinID, TheCall)) diff --git a/clang/test/CodeGen/builtins.c b/clang/test/CodeGen/builtins.c index 4f9641d357b7..407e0857d223 100644 --- a/clang/test/CodeGen/builtins.c +++ b/clang/test/CodeGen/builtins.c @@ -983,4 +983,208 @@ void test_builtin_popcountg(unsigned char uc, unsigned short us, // CHECK-NEXT: ret void } +// CHECK-LABEL: define{{.*}} void @test_builtin_clzg +void test_builtin_clzg(unsigned char uc, unsigned short us, unsigned int ui, + unsigned long ul, unsigned long long ull, + unsigned __int128 ui128, unsigned _BitInt(128) ubi128, + signed char sc, short s, int i) { + volatile int lz; + lz = __builtin_clzg(uc); + // CHECK: %1 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %2 = call i8 @llvm.ctlz.i8(i8 %1, i1 true) + // CHECK-NEXT: %cast = sext i8 %2 to i32 + // CHECK-NEXT: store volatile i32 %cast, ptr %lz, align 4 + lz = __builtin_clzg(us); + // CHECK-NEXT: %3 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %4 = call i16 @llvm.ctlz.i16(i16 %3, i1 true) + // CHECK-NEXT: %cast1 = sext i16 %4 to i32 + // CHECK-NEXT: store volatile i32 %cast1, ptr %lz, align 4 + lz = __builtin_clzg(ui); + // CHECK-NEXT: %5 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %6 = call i32 @llvm.ctlz.i32(i32 %5, i1 true) + // CHECK-NEXT: store volatile i32 %6, ptr %lz, align 4 + lz = __builtin_clzg(ul); + // CHECK-NEXT: %7 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %8 = call i64 @llvm.ctlz.i64(i64 %7, i1 true) + // CHECK-NEXT: %cast2 = trunc i64 %8 to i32 + // CHECK-NEXT: store volatile i32 %cast2, ptr %lz, align 4 + lz = __builtin_clzg(ull); + // CHECK-NEXT: %9 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %10 = call i64 @llvm.ctlz.i64(i64 %9, i1 true) + // CHECK-NEXT: %cast3 = trunc i64 %10 to i32 + // CHECK-NEXT: store volatile i32 %cast3, ptr %lz, align 4 + lz = __builtin_clzg(ui128); + // CHECK-NEXT: %11 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %12 = call i128 @llvm.ctlz.i128(i128 %11, i1 true) + // CHECK-NEXT: %cast4 = trunc i128 %12 to i32 + // CHECK-NEXT: store volatile i32 %cast4, ptr %lz, align 4 + lz = __builtin_clzg(ubi128); + // CHECK-NEXT: %13 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %14 = call i128 @llvm.ctlz.i128(i128 %13, i1 true) + // CHECK-NEXT: %cast5 = trunc i128 %14 to i32 + // CHECK-NEXT: store volatile i32 %cast5, ptr %lz, align 4 + lz = __builtin_clzg(uc, sc); + // CHECK-NEXT: %15 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %16 = call i8 @llvm.ctlz.i8(i8 %15, i1 true) + // CHECK-NEXT: %cast6 = sext i8 %16 to i32 + // CHECK-NEXT: %iszero = icmp eq i8 %15, 0 + // CHECK-NEXT: %17 = load i8, ptr %sc.addr, align 1 + // CHECK-NEXT: %conv = sext i8 %17 to i32 + // CHECK-NEXT: %clzg = select i1 %iszero, i32 %conv, i32 %cast6 + // CHECK-NEXT: store volatile i32 %clzg, ptr %lz, align 4 + lz = __builtin_clzg(us, uc); + // CHECK-NEXT: %18 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %19 = call i16 @llvm.ctlz.i16(i16 %18, i1 true) + // CHECK-NEXT: %cast7 = sext i16 %19 to i32 + // CHECK-NEXT: %iszero8 = icmp eq i16 %18, 0 + // CHECK-NEXT: %20 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %conv9 = zext i8 %20 to i32 + // CHECK-NEXT: %clzg10 = select i1 %iszero8, i32 %conv9, i32 %cast7 + // CHECK-NEXT: store volatile i32 %clzg10, ptr %lz, align 4 + lz = __builtin_clzg(ui, s); + // CHECK-NEXT: %21 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %22 = call i32 @llvm.ctlz.i32(i32 %21, i1 true) + // CHECK-NEXT: %iszero11 = icmp eq i32 %21, 0 + // CHECK-NEXT: %23 = load i16, ptr %s.addr, align 2 + // CHECK-NEXT: %conv12 = sext i16 %23 to i32 + // CHECK-NEXT: %clzg13 = select i1 %iszero11, i32 %conv12, i32 %22 + // CHECK-NEXT: store volatile i32 %clzg13, ptr %lz, align 4 + lz = __builtin_clzg(ul, us); + // CHECK-NEXT: %24 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %25 = call i64 @llvm.ctlz.i64(i64 %24, i1 true) + // CHECK-NEXT: %cast14 = trunc i64 %25 to i32 + // CHECK-NEXT: %iszero15 = icmp eq i64 %24, 0 + // CHECK-NEXT: %26 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %conv16 = zext i16 %26 to i32 + // CHECK-NEXT: %clzg17 = select i1 %iszero15, i32 %conv16, i32 %cast14 + // CHECK-NEXT: store volatile i32 %clzg17, ptr %lz, align 4 + lz = __builtin_clzg(ull, i); + // CHECK-NEXT: %27 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %28 = call i64 @llvm.ctlz.i64(i64 %27, i1 true) + // CHECK-NEXT: %cast18 = trunc i64 %28 to i32 + // CHECK-NEXT: %iszero19 = icmp eq i64 %27, 0 + // CHECK-NEXT: %29 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg20 = select i1 %iszero19, i32 %29, i32 %cast18 + // CHECK-NEXT: store volatile i32 %clzg20, ptr %lz, align 4 + lz = __builtin_clzg(ui128, i); + // CHECK-NEXT: %30 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %31 = call i128 @llvm.ctlz.i128(i128 %30, i1 true) + // CHECK-NEXT: %cast21 = trunc i128 %31 to i32 + // CHECK-NEXT: %iszero22 = icmp eq i128 %30, 0 + // CHECK-NEXT: %32 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg23 = select i1 %iszero22, i32 %32, i32 %cast21 + // CHECK-NEXT: store volatile i32 %clzg23, ptr %lz, align 4 + lz = __builtin_clzg(ubi128, i); + // CHECK-NEXT: %33 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %34 = call i128 @llvm.ctlz.i128(i128 %33, i1 true) + // CHECK-NEXT: %cast24 = trunc i128 %34 to i32 + // CHECK-NEXT: %iszero25 = icmp eq i128 %33, 0 + // CHECK-NEXT: %35 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg26 = select i1 %iszero25, i32 %35, i32 %cast24 + // CHECK-NEXT: store volatile i32 %clzg26, ptr %lz, align 4 + // CHECK-NEXT: ret void +} + +// CHECK-LABEL: define{{.*}} void @test_builtin_ctzg +void test_builtin_ctzg(unsigned char uc, unsigned short us, unsigned int ui, + unsigned long ul, unsigned long long ull, + unsigned __int128 ui128, unsigned _BitInt(128) ubi128, + signed char sc, short s, int i) { + volatile int tz; + tz = __builtin_ctzg(uc); + // CHECK: %1 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %2 = call i8 @llvm.cttz.i8(i8 %1, i1 true) + // CHECK-NEXT: %cast = sext i8 %2 to i32 + // CHECK-NEXT: store volatile i32 %cast, ptr %tz, align 4 + tz = __builtin_ctzg(us); + // CHECK-NEXT: %3 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %4 = call i16 @llvm.cttz.i16(i16 %3, i1 true) + // CHECK-NEXT: %cast1 = sext i16 %4 to i32 + // CHECK-NEXT: store volatile i32 %cast1, ptr %tz, align 4 + tz = __builtin_ctzg(ui); + // CHECK-NEXT: %5 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %6 = call i32 @llvm.cttz.i32(i32 %5, i1 true) + // CHECK-NEXT: store volatile i32 %6, ptr %tz, align 4 + tz = __builtin_ctzg(ul); + // CHECK-NEXT: %7 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %8 = call i64 @llvm.cttz.i64(i64 %7, i1 true) + // CHECK-NEXT: %cast2 = trunc i64 %8 to i32 + // CHECK-NEXT: store volatile i32 %cast2, ptr %tz, align 4 + tz = __builtin_ctzg(ull); + // CHECK-NEXT: %9 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %10 = call i64 @llvm.cttz.i64(i64 %9, i1 true) + // CHECK-NEXT: %cast3 = trunc i64 %10 to i32 + // CHECK-NEXT: store volatile i32 %cast3, ptr %tz, align 4 + tz = __builtin_ctzg(ui128); + // CHECK-NEXT: %11 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %12 = call i128 @llvm.cttz.i128(i128 %11, i1 true) + // CHECK-NEXT: %cast4 = trunc i128 %12 to i32 + // CHECK-NEXT: store volatile i32 %cast4, ptr %tz, align 4 + tz = __builtin_ctzg(ubi128); + // CHECK-NEXT: %13 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %14 = call i128 @llvm.cttz.i128(i128 %13, i1 true) + // CHECK-NEXT: %cast5 = trunc i128 %14 to i32 + // CHECK-NEXT: store volatile i32 %cast5, ptr %tz, align 4 + tz = __builtin_ctzg(uc, sc); + // CHECK-NEXT: %15 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %16 = call i8 @llvm.cttz.i8(i8 %15, i1 true) + // CHECK-NEXT: %cast6 = sext i8 %16 to i32 + // CHECK-NEXT: %iszero = icmp eq i8 %15, 0 + // CHECK-NEXT: %17 = load i8, ptr %sc.addr, align 1 + // CHECK-NEXT: %conv = sext i8 %17 to i32 + // CHECK-NEXT: %ctzg = select i1 %iszero, i32 %conv, i32 %cast6 + // CHECK-NEXT: store volatile i32 %ctzg, ptr %tz, align 4 + tz = __builtin_ctzg(us, uc); + // CHECK-NEXT: %18 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %19 = call i16 @llvm.cttz.i16(i16 %18, i1 true) + // CHECK-NEXT: %cast7 = sext i16 %19 to i32 + // CHECK-NEXT: %iszero8 = icmp eq i16 %18, 0 + // CHECK-NEXT: %20 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %conv9 = zext i8 %20 to i32 + // CHECK-NEXT: %ctzg10 = select i1 %iszero8, i32 %conv9, i32 %cast7 + // CHECK-NEXT: store volatile i32 %ctzg10, ptr %tz, align 4 + tz = __builtin_ctzg(ui, s); + // CHECK-NEXT: %21 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %22 = call i32 @llvm.cttz.i32(i32 %21, i1 true) + // CHECK-NEXT: %iszero11 = icmp eq i32 %21, 0 + // CHECK-NEXT: %23 = load i16, ptr %s.addr, align 2 + // CHECK-NEXT: %conv12 = sext i16 %23 to i32 + // CHECK-NEXT: %ctzg13 = select i1 %iszero11, i32 %conv12, i32 %22 + // CHECK-NEXT: store volatile i32 %ctzg13, ptr %tz, align 4 + tz = __builtin_ctzg(ul, us); + // CHECK-NEXT: %24 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %25 = call i64 @llvm.cttz.i64(i64 %24, i1 true) + // CHECK-NEXT: %cast14 = trunc i64 %25 to i32 + // CHECK-NEXT: %iszero15 = icmp eq i64 %24, 0 + // CHECK-NEXT: %26 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %conv16 = zext i16 %26 to i32 + // CHECK-NEXT: %ctzg17 = select i1 %iszero15, i32 %conv16, i32 %cast14 + // CHECK-NEXT: store volatile i32 %ctzg17, ptr %tz, align 4 + tz = __builtin_ctzg(ull, i); + // CHECK-NEXT: %27 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %28 = call i64 @llvm.cttz.i64(i64 %27, i1 true) + // CHECK-NEXT: %cast18 = trunc i64 %28 to i32 + // CHECK-NEXT: %iszero19 = icmp eq i64 %27, 0 + // CHECK-NEXT: %29 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg20 = select i1 %iszero19, i32 %29, i32 %cast18 + // CHECK-NEXT: store volatile i32 %ctzg20, ptr %tz, align 4 + tz = __builtin_ctzg(ui128, i); + // CHECK-NEXT: %30 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %31 = call i128 @llvm.cttz.i128(i128 %30, i1 true) + // CHECK-NEXT: %cast21 = trunc i128 %31 to i32 + // CHECK-NEXT: %iszero22 = icmp eq i128 %30, 0 + // CHECK-NEXT: %32 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg23 = select i1 %iszero22, i32 %32, i32 %cast21 + // CHECK-NEXT: store volatile i32 %ctzg23, ptr %tz, align 4 + tz = __builtin_ctzg(ubi128, i); + // CHECK-NEXT: %33 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %34 = call i128 @llvm.cttz.i128(i128 %33, i1 true) + // CHECK-NEXT: %cast24 = trunc i128 %34 to i32 + // CHECK-NEXT: %iszero25 = icmp eq i128 %33, 0 + // CHECK-NEXT: %35 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg26 = select i1 %iszero25, i32 %35, i32 %cast24 + // CHECK-NEXT: store volatile i32 %ctzg26, ptr %tz, align 4 + // CHECK-NEXT: ret void +} + #endif diff --git a/clang/test/CodeGen/ubsan-builtin-checks.c b/clang/test/CodeGen/ubsan-builtin-checks.c index 2bc32d8df485..c7f6078f903b 100644 --- a/clang/test/CodeGen/ubsan-builtin-checks.c +++ b/clang/test/CodeGen/ubsan-builtin-checks.c @@ -23,6 +23,9 @@ void check_ctz(int n) { // CHECK: call void @__ubsan_handle_invalid_builtin __builtin_ctzll(n); + + // CHECK: call void @__ubsan_handle_invalid_builtin + __builtin_ctzg((unsigned int)n); } // CHECK: define{{.*}} void @check_clz @@ -44,4 +47,7 @@ void check_clz(int n) { // CHECK: call void @__ubsan_handle_invalid_builtin __builtin_clzll(n); + + // CHECK: call void @__ubsan_handle_invalid_builtin + __builtin_clzg((unsigned int)n); } diff --git a/clang/test/Sema/builtin-popcountg.c b/clang/test/Sema/builtin-popcountg.c deleted file mode 100644 index 9d095927d24e..000000000000 --- a/clang/test/Sema/builtin-popcountg.c +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: %clang_cc1 -std=c23 -triple=x86_64-pc-linux-gnu -fsyntax-only -verify -Wpedantic %s - -typedef int int2 __attribute__((ext_vector_type(2))); - -void test_builtin_popcountg(short s, int i, __int128 i128, _BitInt(128) bi128, - double d, int2 i2) { - __builtin_popcountg(); - // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} - __builtin_popcountg(i, i); - // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} - __builtin_popcountg(s); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'short')}} - __builtin_popcountg(i); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'int')}} - __builtin_popcountg(i128); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was '__int128')}} - __builtin_popcountg(bi128); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was '_BitInt(128)')}} - __builtin_popcountg(d); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'double')}} - __builtin_popcountg(i2); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'int2' (vector of 2 'int' values))}} -} diff --git a/clang/test/Sema/count-builtins.c b/clang/test/Sema/count-builtins.c new file mode 100644 index 000000000000..79fa812f3f20 --- /dev/null +++ b/clang/test/Sema/count-builtins.c @@ -0,0 +1,87 @@ +// RUN: %clang_cc1 -std=c23 -triple=x86_64-pc-linux-gnu -fsyntax-only -verify -Wpedantic %s + +typedef int int2 __attribute__((ext_vector_type(2))); + +void test_builtin_popcountg(short s, int i, __int128 i128, _BitInt(128) bi128, + double d, int2 i2) { + __builtin_popcountg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_popcountg(i, i); + // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} + __builtin_popcountg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_popcountg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_popcountg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_popcountg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_popcountg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_popcountg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} +} + +void test_builtin_clzg(short s, int i, unsigned int ui, __int128 i128, + _BitInt(128) bi128, double d, int2 i2) { + __builtin_clzg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_clzg(i, i, i); + // expected-error@-1 {{too many arguments to function call, expected at most 2, have 3}} + __builtin_clzg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_clzg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_clzg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_clzg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_clzg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_clzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_clzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_clzg(ui, ui); + // expected-error@-1 {{2nd argument must be an 'int' (was 'unsigned int')}} + __builtin_clzg(ui, i128); + // expected-error@-1 {{2nd argument must be an 'int' (was '__int128')}} + __builtin_clzg(ui, bi128); + // expected-error@-1 {{2nd argument must be an 'int' (was '_BitInt(128)')}} + __builtin_clzg(ui, d); + // expected-error@-1 {{2nd argument must be an 'int' (was 'double')}} + __builtin_clzg(ui, i2); + // expected-error@-1 {{2nd argument must be an 'int' (was 'int2' (vector of 2 'int' values))}} +} + +void test_builtin_ctzg(short s, int i, unsigned int ui, __int128 i128, + _BitInt(128) bi128, double d, int2 i2) { + __builtin_ctzg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_ctzg(i, i, i); + // expected-error@-1 {{too many arguments to function call, expected at most 2, have 3}} + __builtin_ctzg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_ctzg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_ctzg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_ctzg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_ctzg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_ctzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_ctzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_ctzg(ui, ui); + // expected-error@-1 {{2nd argument must be an 'int' (was 'unsigned int')}} + __builtin_ctzg(ui, i128); + // expected-error@-1 {{2nd argument must be an 'int' (was '__int128')}} + __builtin_ctzg(ui, bi128); + // expected-error@-1 {{2nd argument must be an 'int' (was '_BitInt(128)')}} + __builtin_ctzg(ui, d); + // expected-error@-1 {{2nd argument must be an 'int' (was 'double')}} + __builtin_ctzg(ui, i2); + // expected-error@-1 {{2nd argument must be an 'int' (was 'int2' (vector of 2 'int' values))}} +} -- GitLab From 6eff53b4f07c1d8f6ae271254499ec087f40cc83 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 21 Mar 2024 09:35:18 -0700 Subject: [PATCH 076/933] [libc][stdio] implement rename via SYS_renameat2 (#86140) SYS_rename may be unavailable on architectures such as aarch64 and riscv. rename can be implemented in terms of SYS_rename, SYS_renameat, or SYS_renameat2. I don't have a full picture of the history here, but it seems that SYS_renameat might also be unavailable on some platforms. `man 2 rename` mentions that SYS_renameat2 was added in Linux 3.15. We don't need to support such ancient kernel versions prior. Link: #84980 Link: #85068 --- libc/src/stdio/linux/rename.cpp | 4 +++- libc/test/src/stdio/rename_test.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libc/src/stdio/linux/rename.cpp b/libc/src/stdio/linux/rename.cpp index f3d684249ad2..379a6ef3c0c8 100644 --- a/libc/src/stdio/linux/rename.cpp +++ b/libc/src/stdio/linux/rename.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "src/stdio/rename.h" +#include "include/llvm-libc-macros/linux/fcntl-macros.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" #include "src/errno/libc_errno.h" @@ -15,7 +16,8 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(int, rename, (const char *oldpath, const char *newpath)) { - int ret = LIBC_NAMESPACE::syscall_impl(SYS_rename, oldpath, newpath); + int ret = LIBC_NAMESPACE::syscall_impl(SYS_renameat2, AT_FDCWD, oldpath, + AT_FDCWD, newpath, 0); if (ret >= 0) return 0; diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp index 3ed39fe8c0eb..a5dd734c6361 100644 --- a/libc/test/src/stdio/rename_test.cpp +++ b/libc/test/src/stdio/rename_test.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "include/llvm-libc-macros/linux/unistd-macros.h" #include "include/llvm-libc-macros/linux/sys-stat-macros.h" +#include "include/llvm-libc-macros/linux/unistd-macros.h" #include "src/errno/libc_errno.h" #include "src/fcntl/open.h" #include "src/stdio/rename.h" -- GitLab From 69429276098df2f2cf67dcab1c96ce8f56280c11 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 16:37:27 +0000 Subject: [PATCH 077/933] [DAG] combineConcatVectorOfScalars - stop always creating UNDEF nodes. NFC. Noticed in debug logs - most calls to visitVECTOR_SHUFFLE resulted into wasteful UNDEF node creations, despite almost never being used. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c83793d15b2e..7009f375df11 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -23447,9 +23447,7 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { SDLoc DL(N); EVT VT = N->getValueType(0); SmallVector Ops; - EVT SVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); - SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); // Keep track of what we encounter. bool AnyInteger = false; @@ -23459,7 +23457,7 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { !Op.getOperand(0).getValueType().isVector()) Ops.push_back(Op.getOperand(0)); else if (ISD::UNDEF == Op.getOpcode()) - Ops.push_back(ScalarUndef); + Ops.push_back(DAG.getNode(ISD::UNDEF, DL, SVT)); else return SDValue(); @@ -23479,13 +23477,12 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { // Replace UNDEFs by another scalar UNDEF node, of the final desired type. if (AnyFP) { SVT = EVT::getFloatingPointVT(OpVT.getSizeInBits()); - ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); if (AnyInteger) { for (SDValue &Op : Ops) { if (Op.getValueType() == SVT) continue; if (Op.isUndef()) - Op = ScalarUndef; + Op = DAG.getNode(ISD::UNDEF, DL, SVT); else Op = DAG.getBitcast(SVT, Op); } -- GitLab From 4bf8dc1a0f9546afb2c13c121e34237ce16cfca6 Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Thu, 21 Mar 2024 17:48:13 +0100 Subject: [PATCH 078/933] [libc++] Remove macros for keeping std::allocator members and void specialization after C++20 (#85806) Fixes #75975. Remove `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS` for the LLVM 19 release, it was previously marked as deprecated in LLVM 18. I believe that `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION` was only used by Google in conjunction with `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`. Removing both macros together should not cause any issues in practice, even though we did not announce the removal of `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION` before. --- libcxx/docs/ReleaseNotes/19.rst | 3 +- libcxx/docs/UsingLibcxx.rst | 12 --- libcxx/include/__config | 2 - libcxx/include/__memory/allocator.h | 16 ++-- ....cxx2a.pass.cpp => address.cxx20.pass.cpp} | 33 ++++---- .../address.cxx20.verify.cpp | 42 ++++++++++ .../address.depr_in_cxx17.verify.cpp | 4 +- ...cxx2a.pass.cpp => allocate.cxx20.pass.cpp} | 31 ++++---- .../allocate.cxx20.verify.cpp | 23 ++++++ .../allocate.cxx2a.verify.cpp | 28 ------- .../allocate.depr_in_cxx17.verify.cpp | 4 +- ...xx2a.pass.cpp => construct.cxx20.pass.cpp} | 53 ++++++------- .../construct.cxx20.verify.cpp | 77 +++++++++++++++++++ ...cxx2a.pass.cpp => max_size.cxx20.pass.cpp} | 15 ++-- .../max_size.cxx20.verify.cpp | 32 ++++++++ ...ass.cpp => allocator_types.cxx20.pass.cpp} | 24 +++--- ...ed_allocator_members.deprecated.verify.cpp | 20 ----- ...cxx20_allocator_void_no_members.verify.cpp | 25 ------ ...xx20_with_removed_members.compile.pass.cpp | 22 ------ .../containers/sequences/deque/types.pass.cpp | 25 +++--- .../containers/sequences/list/types.pass.cpp | 13 ++-- .../sequences/vector/types.pass.cpp | 25 +++--- 22 files changed, 291 insertions(+), 238 deletions(-) rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{address.cxx2a.pass.cpp => address.cxx20.pass.cpp} (52%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{allocate.cxx2a.pass.cpp => allocate.cxx20.pass.cpp} (82%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{construct.cxx2a.pass.cpp => construct.cxx20.pass.cpp} (75%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{max_size.cxx2a.pass.cpp => max_size.cxx20.pass.cpp} (56%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/{allocator_types.cxx2a.pass.cpp => allocator_types.cxx20.pass.cpp} (59%) delete mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp delete mode 100644 libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp delete mode 100644 libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index c70ae477fafc..cac42f9c3c3f 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -70,7 +70,8 @@ Deprecations and Removals - The ``_LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT`` macro that changed the behavior for narrowing conversions in ``std::variant`` has been removed in LLVM 19. -- TODO: The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` macro has been removed in LLVM 19. +- The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION`` + macros have been removed in LLVM 19. - TODO: The ``_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES`` macros have been removed in LLVM 19. C++17 and C++20 removed features can still be re-enabled individually. diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst index 3b1be286c169..ac12b0b96950 100644 --- a/libcxx/docs/UsingLibcxx.rst +++ b/libcxx/docs/UsingLibcxx.rst @@ -244,18 +244,6 @@ C++20 Specific Configuration Macros This macro is deprecated and will be removed in LLVM-19. Use the individual macros listed below. -**_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS**: - This macro is used to re-enable redundant members of `allocator`, - including `pointer`, `reference`, `rebind`, `address`, `max_size`, - `construct`, `destroy`, and the two-argument overload of `allocate`. - This macro has been deprecated and will be removed in LLVM-19. - -**_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION**: - This macro is used to re-enable the library-provided specializations of - `allocator` and `allocator`. - Use it in conjunction with `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS` - to ensure that removed members of `allocator` can be accessed. - **_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS**: This macro is used to re-enable the `argument_type`, `result_type`, `first_argument_type`, and `second_argument_type` members of class diff --git a/libcxx/include/__config b/libcxx/include/__config index 11e13e0c2498..132cace3d5b2 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1238,8 +1238,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c # endif // _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES # if defined(_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES) -# define _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -# define _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION # define _LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS # define _LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS # define _LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h index 4e6303914c38..26e5d4978b15 100644 --- a/libcxx/include/__memory/allocator.h +++ b/libcxx/include/__memory/allocator.h @@ -31,18 +31,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class allocator; -#if defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) -# pragma clang deprecated( \ - _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS, \ - "_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS is deprecated in LLVM 18 and will be removed in LLVM 19") -#endif - -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION) +#if _LIBCPP_STD_VER <= 17 // These specializations shouldn't be marked _LIBCPP_DEPRECATED_IN_CXX17. // Specializing allocator is deprecated, but not using it. template <> class _LIBCPP_TEMPLATE_VIS allocator { -# if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +# if _LIBCPP_STD_VER <= 17 public: _LIBCPP_DEPRECATED_IN_CXX17 typedef void* pointer; @@ -58,7 +52,7 @@ public: template <> class _LIBCPP_TEMPLATE_VIS allocator { -# if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +# if _LIBCPP_STD_VER <= 17 public: _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* pointer; @@ -141,7 +135,7 @@ public: } // C++20 Removed members -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +#if _LIBCPP_STD_VER <= 17 _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp* pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp& reference; @@ -221,7 +215,7 @@ public: } // C++20 Removed members -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +#if _LIBCPP_STD_VER <= 17 _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& reference; diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp similarity index 52% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp index 59657ca46a14..d9a65eee4c13 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp @@ -12,11 +12,9 @@ // pointer address(reference x) const; // const_pointer address(const_reference x) const; -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. +// Removed in C++20, deprecated in C++17. -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -25,25 +23,22 @@ #include "test_macros.h" template -void test_address() -{ - T* tp = new T(); - const T* ctp = tp; - const std::allocator a; - assert(a.address(*tp) == tp); - assert(a.address(*ctp) == tp); - delete tp; +void test_address() { + T* tp = new T(); + const T* ctp = tp; + const std::allocator a; + assert(a.address(*tp) == tp); + assert(a.address(*ctp) == tp); + delete tp; } -struct A -{ - void operator&() const {} +struct A { + void operator&() const {} }; -int main(int, char**) -{ - test_address(); - test_address(); +int main(int, char**) { + test_address(); + test_address(); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp new file mode 100644 index 000000000000..21fd4d23449b --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// pointer address(reference x) const; +// const_pointer address(const_reference x) const; + +// In C++20, parts of std::allocator have been removed. +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include + +#include "test_macros.h" + +template +void test_address() { + T* tp = new T(); + const T* ctp = tp; + const std::allocator a; + assert(a.address(*tp) == tp); // expected-error 2 {{no member}} + assert(a.address(*ctp) == tp); // expected-error 2 {{no member}} + delete tp; +} + +struct A { + void operator&() const {} +}; + +int main(int, char**) { + test_address(); + test_address(); + + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp index 83d059a838ff..4098bdb2ee92 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp @@ -14,9 +14,7 @@ // Deprecated in C++17 -// UNSUPPORTED: c++03, c++11, c++14 - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -Wno-deprecated-pragma +// REQUIRES: c++17 #include diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp similarity index 82% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp index f2fb606ee6db..8fc6628ebfba 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp @@ -11,17 +11,18 @@ // allocator: // T* allocate(size_t n, const void* hint); -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// Removed in C++20, deprecated in C++17. + +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include #include -#include // for std::max_align_t +#include // for std::max_align_t #include "test_macros.h" #include "count_new.h" - #ifdef TEST_HAS_NO_ALIGNED_ALLOCATION static const bool UsingAlignedNew = false; #else @@ -36,7 +37,6 @@ static const std::size_t MaxAligned = std::alignment_of::value static const std::size_t OverAligned = MaxAligned * 2; - template struct TEST_ALIGNAS(Align) AlignedType { char data; @@ -48,7 +48,6 @@ struct TEST_ALIGNAS(Align) AlignedType { template int AlignedType::constructed = 0; - template void test_aligned() { typedef AlignedType T; @@ -56,11 +55,11 @@ void test_aligned() { globalMemCounter.reset(); std::allocator a; const bool IsOverAlignedType = Align > MaxAligned; - const bool ExpectAligned = IsOverAlignedType && UsingAlignedNew; + const bool ExpectAligned = IsOverAlignedType && UsingAlignedNew; { - globalMemCounter.last_new_size = 0; + globalMemCounter.last_new_size = 0; globalMemCounter.last_new_align = 0; - T* ap2 = a.allocate(11, (const void*)5); + T* ap2 = a.allocate(11, (const void*)5); DoNotOptimize(ap2); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkNewCalledEq(1)); @@ -80,14 +79,14 @@ void test_aligned() { } int main(int, char**) { - test_aligned<1>(); - test_aligned<2>(); - test_aligned<4>(); - test_aligned<8>(); - test_aligned<16>(); - test_aligned(); - test_aligned(); - test_aligned(); + test_aligned<1>(); + test_aligned<2>(); + test_aligned<4>(); + test_aligned<8>(); + test_aligned<16>(); + test_aligned(); + test_aligned(); + test_aligned(); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp new file mode 100644 index 000000000000..bf02c7570d39 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// + +// allocator: +// T* allocate(size_t n, const void* hint); + +// Removed in C++20. + +#include + +void f() { + std::allocator a; + a.allocate(3, nullptr); // expected-error {{too many arguments to function call}} +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp deleted file mode 100644 index 2289cd6cd404..000000000000 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// allocator: -// T* allocate(size_t n, const void* hint); - -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - -#include - -void f() { - std::allocator a; - a.allocate(3, nullptr); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} -} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp index 8b2e862e9503..8629df3c4164 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp @@ -13,9 +13,7 @@ // Deprecated in C++17 -// UNSUPPORTED: c++03, c++11, c++14 - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -Wno-deprecated-pragma +// REQUIRES: c++17 #include diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp similarity index 75% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp index d3a7dadbbe11..9a37cf8af8e6 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp @@ -11,12 +11,10 @@ // allocator: // template void construct(pointer p, Args&&... args); -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// In C++20, parts of std::allocator have been removed. +// In C++17, they were deprecated. // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// REQUIRES: c++03 || c++11 || c++14 || c++17 #include #include @@ -26,42 +24,39 @@ int A_constructed = 0; -struct A -{ - int data; - A() {++A_constructed;} +struct A { + int data; + A() { ++A_constructed; } - A(const A&) {++A_constructed;} + A(const A&) { ++A_constructed; } - explicit A(int) {++A_constructed;} - A(int, int*) {++A_constructed;} + explicit A(int) { ++A_constructed; } + A(int, int*) { ++A_constructed; } - ~A() {--A_constructed;} + ~A() { --A_constructed; } }; int move_only_constructed = 0; #if TEST_STD_VER >= 11 -class move_only -{ - move_only(const move_only&) = delete; - move_only& operator=(const move_only&)= delete; +class move_only { + move_only(const move_only&) = delete; + move_only& operator=(const move_only&) = delete; public: - move_only(move_only&&) {++move_only_constructed;} - move_only& operator=(move_only&&) {return *this;} + move_only(move_only&&) { ++move_only_constructed; } + move_only& operator=(move_only&&) { return *this; } - move_only() {++move_only_constructed;} - ~move_only() {--move_only_constructed;} + move_only() { ++move_only_constructed; } + ~move_only() { --move_only_constructed; } public: - int data; // unused other than to make sizeof(move_only) == sizeof(int). - // but public to suppress "-Wunused-private-field" + int data; // unused other than to make sizeof(move_only) == sizeof(int). + // but public to suppress "-Wunused-private-field" }; #endif // TEST_STD_VER >= 11 -int main(int, char**) -{ +int main(int, char**) { globalMemCounter.reset(); { std::allocator a; @@ -69,7 +64,7 @@ int main(int, char**) assert(A_constructed == 0); globalMemCounter.last_new_size = 0; - A* ap = a.allocate(3); + A* ap = a.allocate(3); DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkLastNewSizeEq(3 * sizeof(int))); @@ -113,13 +108,13 @@ int main(int, char**) assert(A_constructed == 0); } #if TEST_STD_VER >= 11 - { + { std::allocator a; assert(globalMemCounter.checkOutstandingNewEq(0)); assert(move_only_constructed == 0); globalMemCounter.last_new_size = 0; - move_only* ap = a.allocate(3); + move_only* ap = a.allocate(3); DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkLastNewSizeEq(3 * sizeof(int))); @@ -145,7 +140,7 @@ int main(int, char**) DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(0)); assert(move_only_constructed == 0); - } + } #endif return 0; diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp new file mode 100644 index 000000000000..b39f9d918c95 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// template void construct(pointer p, Args&&... args); + +// Removed in C++20. + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include + +int A_constructed = 0; + +struct A { + int data; + A() { ++A_constructed; } + + A(const A&) { ++A_constructed; } + + explicit A(int) { ++A_constructed; } + A(int, int*) { ++A_constructed; } + + ~A() { --A_constructed; } +}; + +int move_only_constructed = 0; + +class move_only { + move_only(const move_only&) = delete; + move_only& operator=(const move_only&) = delete; + +public: + move_only(move_only&&) { ++move_only_constructed; } + move_only& operator=(move_only&&) { return *this; } + + move_only() { ++move_only_constructed; } + ~move_only() { --move_only_constructed; } + +public: + int data; // unused other than to make sizeof(move_only) == sizeof(int). + // but public to suppress "-Wunused-private-field" +}; + +int main(int, char**) { + { + std::allocator a; + A* ap = a.allocate(3); + a.construct(ap); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, A()); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, 5); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, 5, (int*)0); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.deallocate(ap, 3); + } + { + std::allocator a; + move_only* ap = a.allocate(3); + a.construct(ap); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, move_only()); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.deallocate(ap, 3); + } + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp similarity index 56% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp index b07568355fee..92e3b919b0f7 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp @@ -11,11 +11,9 @@ // allocator: // size_type max_size() const throw(); -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. +// Removed in C++20, deprecated in C++17. -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -27,11 +25,10 @@ int new_called = 0; -int main(int, char**) -{ - const std::allocator a; - std::size_t M = a.max_size(); - assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); +int main(int, char**) { + const std::allocator a; + std::size_t M = a.max_size(); + assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp new file mode 100644 index 000000000000..0e0f3c3f4aa4 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// size_type max_size() const throw(); + +// In C++20, parts of std::allocator have been removed. +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include +#include +#include + +#include "test_macros.h" + +int new_called = 0; + +int main(int, char**) { + const std::allocator a; + std::size_t M = a.max_size(); // expected-error {{no member}} + assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); + + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp similarity index 59% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp index a6134b04a8f5..e462e07d896c 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp @@ -26,7 +26,9 @@ // ... // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// Removed in C++20, deprecated in C++17. + +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -35,17 +37,17 @@ template void test() { - static_assert((std::is_same::size_type, std::size_t>::value), ""); - static_assert((std::is_same::difference_type, std::ptrdiff_t>::value), ""); - static_assert((std::is_same::pointer, T*>::value), ""); - static_assert((std::is_same::const_pointer, const T*>::value), ""); - static_assert((std::is_same::reference, T&>::value), ""); - static_assert((std::is_same::const_reference, const T&>::value), ""); - static_assert((std::is_same::template rebind::other, - std::allocator >::value), ""); + static_assert((std::is_same::size_type, std::size_t>::value), ""); + static_assert((std::is_same::difference_type, std::ptrdiff_t>::value), ""); + static_assert((std::is_same::pointer, T*>::value), ""); + static_assert((std::is_same::const_pointer, const T*>::value), ""); + static_assert((std::is_same::reference, T&>::value), ""); + static_assert((std::is_same::const_reference, const T&>::value), ""); + static_assert( + (std::is_same::template rebind::other, std::allocator >::value), ""); } int main(int, char**) { - test(); - return 0; + test(); + return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp deleted file mode 100644 index ab6495ea9db4..000000000000 --- a/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// Ensure that defining _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS yields a -// deprecation warning. We intend to issue a deprecation warning in LLVM 18 -// and remove the macro entirely in LLVM 19. As such, this test will be quite -// short lived. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS - -// UNSUPPORTED: clang-modules-build - -#include // expected-warning@* 1+ {{macro '_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS' has been marked as deprecated}} diff --git a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp b/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp deleted file mode 100644 index 8888683a044f..000000000000 --- a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp +++ /dev/null @@ -1,25 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// Check that members of std::allocator are not provided in C++20 -// with _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION but without -// _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS. - -// UNSUPPORTED: c++03, c++11, c++14, c++17 -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION -// -// Ignore any extra errors arising from typo correction. -// ADDITIONAL_COMPILE_FLAGS: -Xclang -verify-ignore-unexpected=error - -#include - -std::allocator::pointer x; // expected-error-re {{no {{(type|template)}} named 'pointer'}} -std::allocator::const_pointer y; // expected-error-re {{no {{(type|template)}} named 'const_pointer'}} -std::allocator::value_type z; // expected-error-re {{no {{(type|template)}} named 'value_type'}} -std::allocator::rebind::other t; // expected-error-re {{no {{(type|template)}} named 'rebind'}} diff --git a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp b/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp deleted file mode 100644 index 3f151edefe1c..000000000000 --- a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp +++ /dev/null @@ -1,22 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// Check that the nested types of std::allocator are provided in C++20 -// with a flag that keeps the removed members. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION - -#include -#include - -static_assert((std::is_same::pointer, void*>::value), ""); -static_assert((std::is_same::const_pointer, const void*>::value), ""); -static_assert((std::is_same::value_type, void>::value), ""); -static_assert((std::is_same::rebind::other, std::allocator >::value), ""); diff --git a/libcxx/test/std/containers/sequences/deque/types.pass.cpp b/libcxx/test/std/containers/sequences/deque/types.pass.cpp index bfe4808f863d..8c14de0c7744 100644 --- a/libcxx/test/std/containers/sequences/deque/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/deque/types.pass.cpp @@ -28,9 +28,6 @@ // typedef std::reverse_iterator const_reverse_iterator; // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include #include @@ -47,14 +44,22 @@ test() typedef std::deque C; static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::value_type>::value), ""); static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::size_type>::value), ""); + static_assert( + (std::is_same::difference_type>::value), + ""); + static_assert( + (std::is_same::value_type&>::value), ""); + static_assert((std::is_same::value_type&>::value), + ""); + static_assert((std::is_same::pointer>::value), ""); + static_assert( + (std::is_same::const_pointer>::value), ""); static_assert((std::is_same< typename std::iterator_traits::iterator_category, std::random_access_iterator_tag>::value), ""); diff --git a/libcxx/test/std/containers/sequences/list/types.pass.cpp b/libcxx/test/std/containers/sequences/list/types.pass.cpp index 0c1ca27745ce..8fe31e3949de 100644 --- a/libcxx/test/std/containers/sequences/list/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/list/types.pass.cpp @@ -21,9 +21,6 @@ // typedef typename allocator_type::pointer pointer; // typedef typename allocator_type::const_pointer const_pointer; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include @@ -38,10 +35,12 @@ int main(int, char**) typedef std::list C; static_assert((std::is_same::value), ""); static_assert((std::is_same >::value), ""); - static_assert((std::is_same::reference>::value), ""); - static_assert((std::is_same::const_reference>::value), ""); - static_assert((std::is_same::pointer>::value), ""); - static_assert((std::is_same::const_pointer>::value), ""); + static_assert((std::is_same >::value_type&>::value), ""); + static_assert( + (std::is_same >::value_type&>::value), ""); + static_assert((std::is_same >::pointer>::value), ""); + static_assert( + (std::is_same >::const_pointer>::value), ""); static_assert((std::is_signed::value), ""); static_assert((std::is_unsigned::value), ""); diff --git a/libcxx/test/std/containers/sequences/vector/types.pass.cpp b/libcxx/test/std/containers/sequences/vector/types.pass.cpp index 4bcfbe7c3ea4..f4d7fa088842 100644 --- a/libcxx/test/std/containers/sequences/vector/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/vector/types.pass.cpp @@ -28,9 +28,6 @@ // typedef std::reverse_iterator const_reverse_iterator; // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include #include @@ -52,14 +49,22 @@ test() // blindly pulling typedefs out of the allocator. This is why we can't call // test>() below. static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::value_type>::value), ""); static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::size_type>::value), ""); + static_assert( + (std::is_same::difference_type>::value), + ""); + static_assert( + (std::is_same::value_type&>::value), ""); + static_assert((std::is_same::value_type&>::value), + ""); + static_assert((std::is_same::pointer>::value), ""); + static_assert( + (std::is_same::const_pointer>::value), ""); static_assert((std::is_signed::value), ""); static_assert((std::is_unsigned::value), ""); -- GitLab From 797336b1278cb7a8b788e467f8fbbc11939143a8 Mon Sep 17 00:00:00 2001 From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:19:54 +0000 Subject: [PATCH 079/933] Revert "[AMDGPU] MCExpr-ify MC layer kernel descriptor" (#86151) Reverts llvm/llvm-project#80855 --- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 40 +- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 11 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 193 +++------ .../MCTargetDesc/AMDGPUMCKernelDescriptor.cpp | 32 -- .../MCTargetDesc/AMDGPUMCKernelDescriptor.h | 51 --- .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 404 +++++++----------- .../MCTargetDesc/AMDGPUTargetStreamer.h | 33 +- .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 - .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 88 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 10 +- llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s | 27 -- llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s | 281 ------------ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s | 190 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s | 186 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s | 184 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s | 168 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s | 171 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s | 148 ------- llvm/test/MC/AMDGPU/hsa-tg-split.s | 74 ---- 19 files changed, 304 insertions(+), 1988 deletions(-) delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h delete mode 100644 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-tg-split.s diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 052b231d62a3..72e8b59e0a40 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -22,7 +22,6 @@ #include "AMDKernelCodeT.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "R600AsmPrinter.h" #include "SIMachineFunctionInfo.h" @@ -429,43 +428,38 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( return KernelCodeProperties; } -MCKernelDescriptor -AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF, - const SIProgramInfo &PI) const { +amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor( + const MachineFunction &MF, + const SIProgramInfo &PI) const { const GCNSubtarget &STM = MF.getSubtarget(); const Function &F = MF.getFunction(); const SIMachineFunctionInfo *Info = MF.getInfo(); - MCContext &Ctx = MF.getContext(); - MCKernelDescriptor KernelDescriptor; + amdhsa::kernel_descriptor_t KernelDescriptor; + memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor)); assert(isUInt<32>(PI.ScratchSize)); assert(isUInt<32>(PI.getComputePGMRSrc1(STM))); assert(isUInt<32>(PI.getComputePGMRSrc2())); - KernelDescriptor.group_segment_fixed_size = - MCConstantExpr::create(PI.LDSSize, Ctx); - KernelDescriptor.private_segment_fixed_size = - MCConstantExpr::create(PI.ScratchSize, Ctx); + KernelDescriptor.group_segment_fixed_size = PI.LDSSize; + KernelDescriptor.private_segment_fixed_size = PI.ScratchSize; Align MaxKernArgAlign; - KernelDescriptor.kernarg_size = MCConstantExpr::create( - STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx); + KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign); - KernelDescriptor.compute_pgm_rsrc1 = - MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx); - KernelDescriptor.compute_pgm_rsrc2 = - MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx); - KernelDescriptor.kernel_code_properties = - MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx); + KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM); + KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(); + KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF); assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0); - KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create( - STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx); + if (STM.hasGFX90AInsts()) + KernelDescriptor.compute_pgm_rsrc3 = + CurrentProgramInfo.ComputePGMRSrc3GFX90A; - KernelDescriptor.kernarg_preload = MCConstantExpr::create( - AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0, - Ctx); + if (AMDGPU::hasKernargPreload(STM)) + KernelDescriptor.kernarg_preload = + static_cast(Info->getNumKernargPreloadedSGPRs()); return KernelDescriptor; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h index b8b2718d293e..79326cd3d328 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h @@ -28,12 +28,15 @@ class MCCodeEmitter; class MCOperand; namespace AMDGPU { -struct MCKernelDescriptor; namespace HSAMD { class MetadataStreamer; } } // namespace AMDGPU +namespace amdhsa { +struct kernel_descriptor_t; +} + class AMDGPUAsmPrinter final : public AsmPrinter { private: unsigned CodeObjectVersion; @@ -72,9 +75,9 @@ private: uint16_t getAmdhsaKernelCodeProperties( const MachineFunction &MF) const; - AMDGPU::MCKernelDescriptor - getAmdhsaKernelDescriptor(const MachineFunction &MF, - const SIProgramInfo &PI) const; + amdhsa::kernel_descriptor_t getAmdhsaKernelDescriptor( + const MachineFunction &MF, + const SIProgramInfo &PI) const; void initTargetStreamer(Module &M); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 38850f5acadd..529705479646 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -8,7 +8,6 @@ #include "AMDKernelCodeT.h" #include "MCTargetDesc/AMDGPUMCExpr.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -5418,8 +5417,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (getParser().parseIdentifier(KernelName)) return true; - AMDGPU::MCKernelDescriptor KD = - getDefaultAmdhsaKernelDescriptor(&getSTI(), getContext()); + kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI()); StringSet<> Seen; @@ -5459,111 +5457,89 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return TokError(".amdhsa_ directives cannot be repeated"); SMLoc ValStart = getLoc(); - const MCExpr *ExprVal; - if (getParser().parseExpression(ExprVal)) + int64_t IVal; + if (getParser().parseAbsoluteExpression(IVal)) return true; SMLoc ValEnd = getLoc(); SMRange ValRange = SMRange(ValStart, ValEnd); - int64_t IVal = 0; + if (IVal < 0) + return OutOfRangeError(ValRange); + uint64_t Val = IVal; - bool EvaluatableExpr; - if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) { - if (IVal < 0) - return OutOfRangeError(ValRange); - Val = IVal; - } #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \ - if (!isUInt(Val)) \ + if (!isUInt(VALUE)) \ return OutOfRangeError(RANGE); \ - AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \ - getContext()); - -// Some fields use the parsed value immediately which requires the expression to -// be solvable. -#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \ - if (!(RESOLVED)) \ - return Error(IDRange.Start, "directive should have resolvable expression", \ - IDRange); + AMDHSA_BITS_SET(FIELD, ENTRY, VALUE); if (ID == ".amdhsa_group_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.group_segment_fixed_size = ExprVal; + KD.group_segment_fixed_size = Val; } else if (ID == ".amdhsa_private_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.private_segment_fixed_size = ExprVal; + KD.private_segment_fixed_size = Val; } else if (ID == ".amdhsa_kernarg_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.kernarg_size = ExprVal; + KD.kernarg_size = Val; } else if (ID == ".amdhsa_user_sgpr_count") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); ExplicitUserSGPRCount = Val; } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 4; } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val > getMaxNumUserSGPRs()) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val, ValRange); if (Val) { ImpliedUserSGPRCount += Val; PreloadLength = Val; } } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val >= 1024) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val, ValRange); if (Val) PreloadOffset = Val; } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_queue_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_dispatch_id") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; @@ -5572,39 +5548,34 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, - ExprVal, ValRange); + KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val, + ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_private_segment_size") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 1; } else if (ID == ".amdhsa_wavefront_size32") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); EnableWavefrontSize32 = Val; PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal, - ValRange); + KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + Val, ValRange); } else if (ID == ".amdhsa_uses_dynamic_stack") { PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal, - ValRange); + KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") { if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); } else if (ID == ".amdhsa_enable_private_segment") { if (!hasArchitectedFlatScratch()) return Error( @@ -5612,48 +5583,42 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "directive is not supported without architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_info") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val, ValRange); } else if (ID == ".amdhsa_system_vgpr_workitem_id") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val, ValRange); } else if (ID == ".amdhsa_next_free_vgpr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); VGPRRange = ValRange; NextFreeVGPR = Val; } else if (ID == ".amdhsa_next_free_sgpr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); SGPRRange = ValRange; NextFreeSGPR = Val; } else if (ID == ".amdhsa_accum_offset") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); AccumOffset = Val; } else if (ID == ".amdhsa_reserve_vcc") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!isUInt<1>(Val)) return OutOfRangeError(ValRange); ReserveVCC = Val; } else if (ID == ".amdhsa_reserve_flat_scratch") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 7) return Error(IDRange.Start, "directive requires gfx7+", IDRange); if (hasArchitectedFlatScratch()) @@ -5673,105 +5638,97 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { IDRange); } else if (ID == ".amdhsa_float_round_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange); } else if (ID == ".amdhsa_float_round_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange); } else if (ID == ".amdhsa_float_denorm_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange); } else if (ID == ".amdhsa_float_denorm_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal, + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val, ValRange); } else if (ID == ".amdhsa_dx10_clamp") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val, ValRange); } else if (ID == ".amdhsa_ieee_mode") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val, ValRange); } else if (ID == ".amdhsa_fp16_overflow") { if (IVersion.Major < 9) return Error(IDRange.Start, "directive requires gfx9+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val, ValRange); } else if (ID == ".amdhsa_tg_split") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, - ExprVal, ValRange); + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val, + ValRange); } else if (ID == ".amdhsa_workgroup_processor_mode") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val, ValRange); } else if (ID == ".amdhsa_memory_ordered") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val, ValRange); } else if (ID == ".amdhsa_forward_progress") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val, ValRange); } else if (ID == ".amdhsa_shared_vgpr_count") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10 || IVersion.Major >= 12) return Error(IDRange.Start, "directive requires gfx10 or gfx11", IDRange); SharedVGPRCount = Val; PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, - COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal, + COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, - ExprVal, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val, + ValRange); } else if (ID == ".amdhsa_exception_fp_denorm_src") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, - ExprVal, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val, + ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_overflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_underflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_inexact") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_int_div_zero") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_round_robin_scheduling") { if (IVersion.Major < 12) return Error(IDRange.Start, "directive requires gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal, + COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val, ValRange); } else { return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange); @@ -5798,18 +5755,15 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt( VGPRBlocks)) return OutOfRangeError(VGPRRange); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, MCConstantExpr::create(VGPRBlocks, getContext()), - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks); if (!isUInt( SGPRBlocks)) return OutOfRangeError(SGPRRange); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, MCConstantExpr::create(SGPRBlocks, getContext()), - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, + SGPRBlocks); if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount) return TokError("amdgpu_user_sgpr_count smaller than than implied by " @@ -5820,17 +5774,11 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt(UserSGPRCount)) return TokError("too many user SGPRs enabled"); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()), - COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext()); - - int64_t IVal = 0; - if (!KD.kernarg_size->evaluateAsAbsolute(IVal)) - return TokError("Kernarg size should be resolvable"); - uint64_t kernarg_size = IVal; - if (PreloadLength && kernarg_size && - (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size)) + AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, + UserSGPRCount); + + if (PreloadLength && KD.kernarg_size && + (PreloadLength * 4 + PreloadOffset * 4 > KD.kernarg_size)) return TokError("Kernarg preload length + offset is larger than the " "kernarg segment size"); @@ -5842,11 +5790,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "increments of 4"); if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4)) return TokError("accum_offset exceeds total VGPR allocation"); - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc3, - MCConstantExpr::create(AccumOffset / 4 - 1, getContext()), - COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, - COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, + (AccumOffset / 4 - 1)); } if (IVersion.Major >= 10 && IVersion.Major < 12) { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp deleted file mode 100644 index 0179d575464d..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===--- AMDHSAKernelDescriptor.h -----------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "AMDGPUMCKernelDescriptor.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCExpr.h" - -using namespace llvm; -using namespace llvm::AMDGPU; - -void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value, - uint32_t Shift, uint32_t Mask, - MCContext &Ctx) { - auto Sft = MCConstantExpr::create(Shift, Ctx); - auto Msk = MCConstantExpr::create(Mask, Ctx); - Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx); - Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx), - Ctx); -} - -const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift, - uint32_t Mask, MCContext &Ctx) { - auto Sft = MCConstantExpr::create(Shift, Ctx); - auto Msk = MCConstantExpr::create(Mask, Ctx); - return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft, - Ctx); -} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h deleted file mode 100644 index 71659e642dd7..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h +++ /dev/null @@ -1,51 +0,0 @@ -//===--- AMDGPUMCKernelDescriptor.h ---------------------------*- C++ -*---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -/// \file -/// AMDHSA kernel descriptor MCExpr struct for use in MC layer. Uses -/// AMDHSAKernelDescriptor.h for sizes and constants. -/// -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H -#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H - -#include "llvm/Support/AMDHSAKernelDescriptor.h" - -namespace llvm { -class MCExpr; -class MCContext; -namespace AMDGPU { - -struct MCKernelDescriptor { - const MCExpr *group_segment_fixed_size = nullptr; - const MCExpr *private_segment_fixed_size = nullptr; - const MCExpr *kernarg_size = nullptr; - const MCExpr *compute_pgm_rsrc3 = nullptr; - const MCExpr *compute_pgm_rsrc1 = nullptr; - const MCExpr *compute_pgm_rsrc2 = nullptr; - const MCExpr *kernel_code_properties = nullptr; - const MCExpr *kernarg_preload = nullptr; - - // MCExpr for: - // Dst = Dst & ~Mask - // Dst = Dst | (Value << Shift) - static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, - uint32_t Mask, MCContext &Ctx); - - // MCExpr for: - // return (Src & Mask) >> Shift - static const MCExpr *bits_get(const MCExpr *Src, uint32_t Shift, - uint32_t Mask, MCContext &Ctx); -}; - -} // end namespace AMDGPU -} // end namespace llvm - -#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 3006fcdb9282..4742b0b3e52e 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "AMDGPUTargetStreamer.h" -#include "AMDGPUMCKernelDescriptor.h" #include "AMDGPUPTNote.h" #include "AMDKernelCodeT.h" #include "Utils/AMDGPUBaseInfo.h" @@ -308,142 +307,94 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const MCKernelDescriptor &KD, uint64_t NextVGPR, uint64_t NextSGPR, + const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { IsaVersion IVersion = getIsaVersion(STI.getCPU()); - const MCAsmInfo *MAI = getContext().getAsmInfo(); OS << "\t.amdhsa_kernel " << KernelName << '\n'; - auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask, - StringRef Directive) { - int64_t IVal; - OS << "\t\t" << Directive << ' '; - const MCExpr *pgm_rsrc1_bits = - MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext()); - if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal)) - OS << static_cast(IVal); - else - pgm_rsrc1_bits->print(OS, MAI); - OS << '\n'; - }; - - OS << "\t\t.amdhsa_group_segment_fixed_size "; - KD.group_segment_fixed_size->print(OS, MAI); - OS << '\n'; - - OS << "\t\t.amdhsa_private_segment_fixed_size "; - KD.private_segment_fixed_size->print(OS, MAI); - OS << '\n'; - - OS << "\t\t.amdhsa_kernarg_size "; - KD.kernarg_size->print(OS, MAI); - OS << '\n'; - - PrintField( - KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count"); +#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \ + STREAM << "\t\t" << DIRECTIVE << " " \ + << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n'; + + OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size + << '\n'; + OS << "\t\t.amdhsa_private_segment_fixed_size " + << KD.private_segment_fixed_size << '\n'; + OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n'; + + PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT); if (!hasArchitectedFlatScratch(STI)) - PrintField( - KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - ".amdhsa_user_sgpr_private_segment_buffer"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, - ".amdhsa_user_sgpr_dispatch_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, - ".amdhsa_user_sgpr_queue_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - ".amdhsa_user_sgpr_kernarg_segment_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, - ".amdhsa_user_sgpr_dispatch_id"); + PRINT_FIELD( + OS, ".amdhsa_user_sgpr_private_segment_buffer", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID); if (!hasArchitectedFlatScratch(STI)) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, - ".amdhsa_user_sgpr_flat_scratch_init"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); if (hasKernargPreload(STI)) { - PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT, - amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, - ".amdhsa_user_sgpr_kernarg_preload_length"); - PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT, - amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, - ".amdhsa_user_sgpr_kernarg_preload_offset"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD, + kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD, + kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET); } - PrintField( - KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - ".amdhsa_user_sgpr_private_segment_size"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); if (IVersion.Major >= 10) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - ".amdhsa_wavefront_size32"); + PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32); if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, - ".amdhsa_uses_dynamic_stack"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, - (hasArchitectedFlatScratch(STI) - ? ".amdhsa_enable_private_segment" - : ".amdhsa_system_sgpr_private_segment_wavefront_offset")); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, - ".amdhsa_system_sgpr_workgroup_id_x"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, - ".amdhsa_system_sgpr_workgroup_id_y"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, - ".amdhsa_system_sgpr_workgroup_id_z"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, - ".amdhsa_system_sgpr_workgroup_info"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, - ".amdhsa_system_vgpr_workitem_id"); + PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK); + PRINT_FIELD(OS, + (hasArchitectedFlatScratch(STI) + ? ".amdhsa_enable_private_segment" + : ".amdhsa_system_sgpr_private_segment_wavefront_offset"), + KD, compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO); + PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); // These directives are required. OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n'; OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n'; - if (AMDGPU::isGFX90A(STI)) { - // MCExpr equivalent of taking the (accum_offset + 1) * 4. - const MCExpr *accum_bits = MCKernelDescriptor::bits_get( - KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); - accum_bits = MCBinaryExpr::createAdd( - accum_bits, MCConstantExpr::create(1, getContext()), getContext()); - accum_bits = MCBinaryExpr::createMul( - accum_bits, MCConstantExpr::create(4, getContext()), getContext()); - OS << "\t\t.amdhsa_accum_offset "; - int64_t IVal; - if (accum_bits->evaluateAsAbsolute(IVal)) { - OS << static_cast(IVal); - } else { - accum_bits->print(OS, MAI); - } - OS << '\n'; - } + if (AMDGPU::isGFX90A(STI)) + OS << "\t\t.amdhsa_accum_offset " << + (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4 + << '\n'; if (!ReserveVCC) OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n'; @@ -460,105 +411,74 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( break; } - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, - ".amdhsa_float_round_mode_32"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, - ".amdhsa_float_round_mode_16_64"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, - ".amdhsa_float_denorm_mode_32"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, - ".amdhsa_float_denorm_mode_16_64"); + PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32); + PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64); + PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32); + PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64); if (IVersion.Major < 12) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, - ".amdhsa_dx10_clamp"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, - ".amdhsa_ieee_mode"); - } - if (IVersion.Major >= 9) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, - ".amdhsa_fp16_overflow"); + PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP); + PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE); } + if (IVersion.Major >= 9) + PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL); if (AMDGPU::isGFX90A(STI)) - PrintField(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split"); + PRINT_FIELD(OS, ".amdhsa_tg_split", KD, + compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT); if (IVersion.Major >= 10) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, - ".amdhsa_workgroup_processor_mode"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, - ".amdhsa_memory_ordered"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, - ".amdhsa_forward_progress"); + PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE); + PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED); + PRINT_FIELD(OS, ".amdhsa_forward_progress", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS); } if (IVersion.Major >= 10 && IVersion.Major < 12) { - PrintField(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, - ".amdhsa_shared_vgpr_count"); + PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT); } - if (IVersion.Major >= 12) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, - ".amdhsa_round_robin_scheduling"); - } - PrintField( - KD.compute_pgm_rsrc2, - amdhsa:: - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, - ".amdhsa_exception_fp_ieee_invalid_op"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - ".amdhsa_exception_fp_denorm_src"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa:: - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, - ".amdhsa_exception_fp_ieee_div_zero"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - ".amdhsa_exception_fp_ieee_overflow"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - ".amdhsa_exception_fp_ieee_underflow"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - ".amdhsa_exception_fp_ieee_inexact"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - ".amdhsa_exception_int_div_zero"); + if (IVersion.Major >= 12) + PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN); + PRINT_FIELD( + OS, ".amdhsa_exception_fp_ieee_invalid_op", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION); + PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE); + PRINT_FIELD( + OS, ".amdhsa_exception_fp_ieee_div_zero", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT); + PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO); +#undef PRINT_FIELD OS << "\t.end_amdhsa_kernel\n"; } @@ -915,7 +835,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const MCKernelDescriptor &KernelDescriptor, uint64_t NextVGPR, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { auto &Streamer = getStreamer(); auto &Context = Streamer.getContext(); @@ -933,7 +853,7 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( // Kernel descriptor symbol's type and size are fixed. KernelDescriptorSymbol->setType(ELF::STT_OBJECT); KernelDescriptorSymbol->setSize( - MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context)); + MCConstantExpr::create(sizeof(KernelDescriptor), Context)); // The visibility of the kernel code symbol must be protected or less to allow // static relocations from the kernel descriptor to be used. @@ -941,43 +861,31 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED); Streamer.emitLabel(KernelDescriptorSymbol); - Streamer.emitValue( - KernelDescriptor.group_segment_fixed_size, - sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size)); - Streamer.emitValue( - KernelDescriptor.private_segment_fixed_size, - sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size)); - Streamer.emitValue(KernelDescriptor.kernarg_size, - sizeof(amdhsa::kernel_descriptor_t::kernarg_size)); - - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i) - Streamer.emitInt8(0u); + Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size); + Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size); + Streamer.emitInt32(KernelDescriptor.kernarg_size); + + for (uint8_t Res : KernelDescriptor.reserved0) + Streamer.emitInt8(Res); // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The // expression being created is: // (start of kernel code) - (start of kernel descriptor) // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64. - Streamer.emitValue( - MCBinaryExpr::createSub( - MCSymbolRefExpr::create(KernelCodeSymbol, - MCSymbolRefExpr::VK_AMDGPU_REL64, Context), - MCSymbolRefExpr::create(KernelDescriptorSymbol, - MCSymbolRefExpr::VK_None, Context), - Context), - sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset)); - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i) - Streamer.emitInt8(0u); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3)); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1)); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2)); - Streamer.emitValue( - KernelDescriptor.kernel_code_properties, - sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties)); - Streamer.emitValue(KernelDescriptor.kernarg_preload, - sizeof(amdhsa::kernel_descriptor_t::kernarg_preload)); - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i) - Streamer.emitInt8(0u); + Streamer.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create( + KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context), + MCSymbolRefExpr::create( + KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context), + Context), + sizeof(KernelDescriptor.kernel_code_entry_byte_offset)); + for (uint8_t Res : KernelDescriptor.reserved1) + Streamer.emitInt8(Res); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2); + Streamer.emitInt16(KernelDescriptor.kernel_code_properties); + Streamer.emitInt16(KernelDescriptor.kernarg_preload); + for (uint8_t Res : KernelDescriptor.reserved3) + Streamer.emitInt8(Res); } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h index 706897a5dc1f..5aa80ff578c6 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h @@ -22,13 +22,15 @@ class MCSymbol; class formatted_raw_ostream; namespace AMDGPU { - -struct MCKernelDescriptor; namespace HSAMD { struct Metadata; } } // namespace AMDGPU +namespace amdhsa { +struct kernel_descriptor_t; +} + class AMDGPUTargetStreamer : public MCTargetStreamer { AMDGPUPALMetadata PALMetadata; @@ -92,11 +94,10 @@ public: return true; } - virtual void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) {} + virtual void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {} static StringRef getArchNameFromElfMach(unsigned ElfMach); static unsigned getElfMach(StringRef GPU); @@ -149,11 +150,10 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) override; + void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; }; class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer { @@ -205,11 +205,10 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) override; + void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; }; } #endif diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 14a02b6d8e36..0842a58f794b 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -8,7 +8,6 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp - AMDGPUMCKernelDescriptor.cpp R600InstPrinter.cpp R600MCCodeEmitter.cpp R600MCTargetDesc.cpp diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 4970055c4bdb..6d53f68ace70 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -10,7 +10,6 @@ #include "AMDGPU.h" #include "AMDGPUAsmUtils.h" #include "AMDKernelCodeT.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" @@ -21,7 +20,6 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1217,64 +1215,44 @@ void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, } } -MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, - MCContext &Ctx) { +amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( + const MCSubtargetInfo *STI) { IsaVersion Version = getIsaVersion(STI->getCPU()); - MCKernelDescriptor KD; - const MCExpr *ZeroMCExpr = MCConstantExpr::create(0, Ctx); - const MCExpr *OneMCExpr = MCConstantExpr::create(1, Ctx); - - KD.group_segment_fixed_size = ZeroMCExpr; - KD.private_segment_fixed_size = ZeroMCExpr; - KD.compute_pgm_rsrc1 = ZeroMCExpr; - KD.compute_pgm_rsrc2 = ZeroMCExpr; - KD.compute_pgm_rsrc3 = ZeroMCExpr; - KD.kernarg_size = ZeroMCExpr; - KD.kernel_code_properties = ZeroMCExpr; - KD.kernarg_preload = ZeroMCExpr; - - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, - MCConstantExpr::create(amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE, Ctx), - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Ctx); - if (Version.Major < 12) { - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Ctx); - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Ctx); + amdhsa::kernel_descriptor_t KD; + memset(&KD, 0, sizeof(KD)); + + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, + amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); + if (Version.Major >= 12) { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0); + } else { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1); } - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc2, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Ctx); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); if (Version.Major >= 10) { - if (STI->getFeatureBits().test(FeatureWavefrontSize32)) - MCKernelDescriptor::bits_set( - KD.kernel_code_properties, OneMCExpr, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, Ctx); - if (!STI->getFeatureBits().test(FeatureCuMode)) - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Ctx); - - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Ctx); + AMDHSA_BITS_SET(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, + STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1); + } + if (AMDGPU::isGFX90A(*STI)) { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, + STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); } - if (AMDGPU::isGFX90A(*STI) && STI->getFeatureBits().test(FeatureTgSplit)) - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc3, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx); return KD; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 32b73f1d868d..29ac402d9535 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -26,7 +26,6 @@ struct Align; class Argument; class Function; class GlobalValue; -class MCContext; class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; @@ -35,9 +34,12 @@ class StringRef; class Triple; class raw_ostream; +namespace amdhsa { +struct kernel_descriptor_t; +} + namespace AMDGPU { -struct MCKernelDescriptor; struct IsaVersion; /// Generic target versions emitted by this version of LLVM. @@ -850,8 +852,8 @@ unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc); void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, const MCSubtargetInfo *STI); -MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, - MCContext &Ctx); +amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( + const MCSubtargetInfo *STI); bool isGroupSegment(const GlobalValue *GV); bool isGlobalSegment(const GlobalValue *GV); diff --git a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s deleted file mode 100644 index 4623500987be..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s +++ /dev/null @@ -1,27 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// OBJDUMP: 0000 00000000 0f000000 00000000 00000000 - -.text - -.p2align 8 -.type caller,@function -caller: - s_endpgm - -.rodata - -.p2align 6 -.amdhsa_kernel caller - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) -.end_amdhsa_kernel - -.set callee1.private_seg_size, 4 -.set callee2.private_seg_size, 15 - -// ASM: .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) diff --git a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s deleted file mode 100644 index fab3e893352b..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s +++ /dev/null @@ -1,281 +0,0 @@ -// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a %s 2>&1 | FileCheck --check-prefix=ASM %s - -// Some expression currently require (immediately) solvable expressions, i.e., -// they don't depend on yet-unknown symbolic values. - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type user_sgpr_count,@function -user_sgpr_count: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_count - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_count defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_count - -.p2align 8 -.type user_sgpr_private_segment_buffer,@function -user_sgpr_private_segment_buffer: - s_endpgm - -.amdhsa_kernel user_sgpr_private_segment_buffer - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_private_segment_buffer defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer - -.p2align 8 -.type user_sgpr_kernarg_preload_length,@function -user_sgpr_kernarg_preload_length: - s_endpgm - -.amdhsa_kernel user_sgpr_kernarg_preload_length - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_preload_length defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length defined_boolean - -.p2align 8 -.type user_sgpr_kernarg_preload_offset,@function -user_sgpr_kernarg_preload_offset: - s_endpgm - -.amdhsa_kernel user_sgpr_kernarg_preload_offset - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean - -.p2align 8 -.type user_sgpr_dispatch_ptr,@function -user_sgpr_dispatch_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_dispatch_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_dispatch_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr - -.p2align 8 -.type user_sgpr_queue_ptr,@function -user_sgpr_queue_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_queue_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_queue_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr - -.p2align 8 -.type user_sgpr_kernarg_segment_ptr,@function -user_sgpr_kernarg_segment_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_kernarg_segment_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_segment_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr - -.p2align 8 -.type user_sgpr_dispatch_id,@function -user_sgpr_dispatch_id: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_dispatch_id - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_dispatch_id defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id - -.p2align 8 -.type user_sgpr_flat_scratch_init,@function -user_sgpr_flat_scratch_init: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_flat_scratch_init - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_flat_scratch_init defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init - -.p2align 8 -.type user_sgpr_private_segment_size,@function -user_sgpr_private_segment_size: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_private_segment_size - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_private_segment_size defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size - -.p2align 8 -.type wavefront_size32,@function -wavefront_size32: - s_endpgm - -.p2align 6 -.amdhsa_kernel wavefront_size32 - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_wavefront_size32 defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_wavefront_size32 - -.p2align 8 -.type next_free_vgpr,@function -next_free_vgpr: - s_endpgm - -.p2align 6 -.amdhsa_kernel next_free_vgpr - .amdhsa_next_free_vgpr defined_boolean - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_next_free_vgpr - -.p2align 8 -.type next_free_sgpr,@function -next_free_sgpr: - s_endpgm - -.p2align 6 -.amdhsa_kernel next_free_sgpr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr defined_boolean - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_next_free_sgpr - -.p2align 8 -.type accum_offset,@function -accum_offset: - s_endpgm - -.p2align 6 -.amdhsa_kernel accum_offset - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_accum_offset - -.p2align 8 -.type reserve_vcc,@function -reserve_vcc: - s_endpgm - -.p2align 6 -.amdhsa_kernel reserve_vcc - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_reserve_vcc defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_reserve_vcc - -.p2align 8 -.type reserve_flat_scratch,@function -reserve_flat_scratch: - s_endpgm - -.p2align 6 -.amdhsa_kernel reserve_flat_scratch - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_reserve_flat_scratch defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_reserve_flat_scratch - -.p2align 8 -.type shared_vgpr_count,@function -shared_vgpr_count: - s_endpgm - -.p2align 6 -.amdhsa_kernel shared_vgpr_count - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_shared_vgpr_count defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_shared_vgpr_count - -.set defined_boolean, 1 - -// ASM: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s deleted file mode 100644 index 95af59c413ae..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s +++ /dev/null @@ -1,190 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0afe4 801f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0afe4 801f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s deleted file mode 100644 index e1107fb69ba4..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s +++ /dev/null @@ -1,186 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0afe4 811f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0afe4 811f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_enable_private_segment 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s deleted file mode 100644 index 449616d35186..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s +++ /dev/null @@ -1,184 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f02fe4 811f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f02fe4 811f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_round_robin_scheduling defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_round_robin_scheduling defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_enable_private_segment 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_round_robin_scheduling 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s deleted file mode 100644 index c7e05441b45f..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s +++ /dev/null @@ -1,168 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s deleted file mode 100644 index 49a5015987a6..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s +++ /dev/null @@ -1,171 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefix=ASM %s - -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s deleted file mode 100644 index b7f89239160f..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s +++ /dev/null @@ -1,148 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0030 0000ac04 81000000 00000000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0070 0000ac04 81000000 00000000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean - .amdhsa_dx10_clamp defined_boolean - .amdhsa_ieee_mode defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_tg_split defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean - .amdhsa_dx10_clamp defined_boolean - .amdhsa_ieee_mode defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_tg_split defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&63)>>0)+1)*4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&65536)>>16 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset 4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 0 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_tg_split 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 -// ASM-NEXT: .amdhsa_exception_int_div_zero 0 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-tg-split.s b/llvm/test/MC/AMDGPU/hsa-tg-split.s deleted file mode 100644 index 5a4d3e2c279c..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-tg-split.s +++ /dev/null @@ -1,74 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// OBJDUMP: Contents of section .rodata -// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000 - -.text -// ASM: .text - -.amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" -// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type minimal,@function -minimal: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel minimal - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel minimal -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset 4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 0 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 0 -// ASM-NEXT: .amdhsa_tg_split 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 -// ASM-NEXT: .amdhsa_exception_int_div_zero 0 -// ASM-NEXT: .end_amdhsa_kernel -- GitLab From c8772940ee4d85b1a4578b3faea7c825300ce59f Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Thu, 21 Mar 2024 18:22:22 +0100 Subject: [PATCH 080/933] [analyzer] Wrap SymbolicRegions by ElementRegions before getting a FieldRegion (#85211) Inside the ExprEngine when we process the initializers, we create a PostInitializer program-point, which will refer to the field being initialized, see `FieldLoc` inside `ExprEngine::ProcessInitializer`. When a constructor (of which we evaluate the initializer-list) is analyzed in top-level context, then the `this` pointer will be represented by a `SymbolicRegion`, (as it should be). This means that we will form a `FieldRegion{SymbolicRegion{.}}` as the initialized region. ```c++ class Bear { public: void brum() const; }; class Door { public: // PostInitializer would refer to "FieldRegion{SymRegion{this}}" // whereas in the store and everywhere else it would be: // "FieldRegion{ELementRegion{SymRegion{Ty*, this}, 0, Ty}". Door() : ptr(nullptr) { ptr->brum(); // Bug } private: Bear* ptr; }; ``` We (as CSA folks) decided to avoid the creation of FieldRegions directly of symbolic regions in the past: https://github.com/llvm/llvm-project/commit/f8643a9b31c4029942f67d4534c9139b45173504 --- In this patch, I propose to also canonicalize it as in the mentioned patch, into this: `FieldRegion{ElementRegion{SymbolicRegion{Ty*, .}, 0, Ty}` This would mean that FieldRegions will/should never simply wrap a SymbolicRegion directly, but rather an ElementRegion that is sitting in between. This patch should have practically no observable effects, as the store (due to the mentioned patch) was made resilient to this issue, but we use `PostInitializer::getLocationValue()` for an alternative reporting, where we faced this issue. Note that in really rare cases it suppresses now dereference bugs, as demonstrated in the test. It is because in the past we failed to follow the region of the PostInitializer inside the StoreSiteFinder visitor - because it was using this code: ```c++ // If this is a post initializer expression, initializing the region, we // should track the initializer expression. if (std::optional PIP = Pred->getLocationAs()) { const MemRegion *FieldReg = (const MemRegion *)PIP->getLocationValue(); if (FieldReg == R) { StoreSite = Pred; InitE = PIP->getInitializer()->getInit(); } } ``` Notice that the equality check didn't pass for the regions I'm canonicalizing in this patch. Given the nature of this change, we would rather upstream this patch. CPP-4954 --- .../Core/PathSensitive/ProgramState.h | 16 ++-------- .../lib/StaticAnalyzer/Core/ProgramState.cpp | 32 +++++++++++++++++++ .../inlining/false-positive-suppression.cpp | 17 ++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h index ca75c2a756a4..51d76dc257ee 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h @@ -494,6 +494,8 @@ private: InvalidatedSymbols *IS, RegionAndSymbolInvalidationTraits *HTraits, const CallEvent *Call) const; + + SVal wrapSymbolicRegion(SVal Base) const; }; //===----------------------------------------------------------------------===// @@ -782,20 +784,6 @@ inline SVal ProgramState::getLValue(const ObjCIvarDecl *D, SVal Base) const { return getStateManager().StoreMgr->getLValueIvar(D, Base); } -inline SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const { - return getStateManager().StoreMgr->getLValueField(D, Base); -} - -inline SVal ProgramState::getLValue(const IndirectFieldDecl *D, - SVal Base) const { - StoreManager &SM = *getStateManager().StoreMgr; - for (const auto *I : D->chain()) { - Base = SM.getLValueField(cast(I), Base); - } - - return Base; -} - inline SVal ProgramState::getLValue(QualType ElementType, SVal Idx, SVal Base) const{ if (std::optional N = Idx.getAs()) return getStateManager().StoreMgr->getLValueElement(ElementType, *N, Base); diff --git a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp index f12f1a5ac970..f82cd944750a 100644 --- a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp +++ b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp @@ -226,6 +226,20 @@ ProgramStateRef ProgramState::killBinding(Loc LV) const { return makeWithStore(newStore); } +/// SymbolicRegions are expected to be wrapped by an ElementRegion as a +/// canonical representation. As a canonical representation, SymbolicRegions +/// should be wrapped by ElementRegions before getting a FieldRegion. +/// See f8643a9b31c4029942f67d4534c9139b45173504 why. +SVal ProgramState::wrapSymbolicRegion(SVal Val) const { + const auto *BaseReg = dyn_cast_or_null(Val.getAsRegion()); + if (!BaseReg) + return Val; + + StoreManager &SM = getStateManager().getStoreManager(); + QualType ElemTy = BaseReg->getPointeeStaticType(); + return loc::MemRegionVal{SM.GetElementZeroRegion(BaseReg, ElemTy)}; +} + ProgramStateRef ProgramState::enterStackFrame(const CallEvent &Call, const StackFrameContext *CalleeCtx) const { @@ -451,6 +465,24 @@ void ProgramState::setStore(const StoreRef &newStore) { store = newStoreStore; } +SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const { + Base = wrapSymbolicRegion(Base); + return getStateManager().StoreMgr->getLValueField(D, Base); +} + +SVal ProgramState::getLValue(const IndirectFieldDecl *D, SVal Base) const { + StoreManager &SM = *getStateManager().StoreMgr; + Base = wrapSymbolicRegion(Base); + + // FIXME: This should work with `SM.getLValueField(D->getAnonField(), Base)`, + // but that would break some tests. There is probably a bug somewhere that it + // would expose. + for (const auto *I : D->chain()) { + Base = SM.getLValueField(cast(I), Base); + } + return Base; +} + //===----------------------------------------------------------------------===// // State pretty-printing. //===----------------------------------------------------------------------===// diff --git a/clang/test/Analysis/inlining/false-positive-suppression.cpp b/clang/test/Analysis/inlining/false-positive-suppression.cpp index 56659b4a1941..2f9ed7f78b3f 100644 --- a/clang/test/Analysis/inlining/false-positive-suppression.cpp +++ b/clang/test/Analysis/inlining/false-positive-suppression.cpp @@ -210,3 +210,20 @@ namespace Cleanups { testArgumentHelper(NonTrivial().getNull()); } } + +class Bear *getNullBear() { return nullptr; } +class Bear { +public: + void brum() const; +}; +class Door { +public: + Door() : ptr(getNullBear()) { + ptr->brum(); +#ifndef SUPPRESSED + // expected-warning@-2 {{Called C++ object pointer is null}} +#endif + } +private: + Bear* ptr; +}; -- GitLab From 86d479fd7c837e97be116ffb9e4c92812b87360f Mon Sep 17 00:00:00 2001 From: T-Gruber <100079402+T-Gruber@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:27:53 +0100 Subject: [PATCH 081/933] Adapted MemRegion::getDescriptiveName to handle ElementRegions (#85104) Fixes https://github.com/llvm/llvm-project/issues/84463 Changes: - Adapted MemRegion::getDescriptiveName - Added unittest to check name for a given clang::ento::ElementRegion - Some format changes due to clang-format --------- Co-authored-by: Andreas Steinhausen Co-authored-by: Balazs Benics --- clang/lib/StaticAnalyzer/Core/MemRegion.cpp | 20 ++- clang/unittests/StaticAnalyzer/CMakeLists.txt | 1 + .../MemRegionDescriptiveNameTest.cpp | 145 ++++++++++++++++++ .../clang/unittests/StaticAnalyzer/BUILD.gn | 1 + 4 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp index 16db6b249dc9..a8e573f7982b 100644 --- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp +++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp @@ -720,13 +720,21 @@ std::string MemRegion::getDescriptiveName(bool UseQuotes) const { CI->getValue().toString(Idx); ArrayIndices = (llvm::Twine("[") + Idx.str() + "]" + ArrayIndices).str(); } - // If not a ConcreteInt, try to obtain the variable - // name by calling 'getDescriptiveName' recursively. + // Index is symbolic, but may have a descriptive name. else { - std::string Idx = ER->getDescriptiveName(false); - if (!Idx.empty()) { - ArrayIndices = (llvm::Twine("[") + Idx + "]" + ArrayIndices).str(); - } + auto SI = ER->getIndex().getAs(); + if (!SI) + return ""; + + const MemRegion *OR = SI->getAsSymbol()->getOriginRegion(); + if (!OR) + return ""; + + std::string Idx = OR->getDescriptiveName(false); + if (Idx.empty()) + return ""; + + ArrayIndices = (llvm::Twine("[") + Idx + "]" + ArrayIndices).str(); } R = ER->getSuperRegion(); } diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt index 775f0f8486b8..519be36fe0fa 100644 --- a/clang/unittests/StaticAnalyzer/CMakeLists.txt +++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt @@ -11,6 +11,7 @@ add_clang_unittest(StaticAnalysisTests CallEventTest.cpp ConflictingEvalCallsTest.cpp FalsePositiveRefutationBRVisitorTest.cpp + MemRegionDescriptiveNameTest.cpp NoStateChangeFuncVisitorTest.cpp ParamRegionTest.cpp RangeSetTest.cpp diff --git a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp new file mode 100644 index 000000000000..ba0c4d25e13b --- /dev/null +++ b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp @@ -0,0 +1,145 @@ +//===- MemRegionDescriptiveNameTest.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CheckerRegistration.h" +#include "clang/StaticAnalyzer/Core/Checker.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h" +#include "gtest/gtest.h" +#include + +using namespace clang; +using namespace ento; + +namespace { + +class DescriptiveNameChecker : public Checker { +public: + void checkPreCall(const CallEvent &Call, CheckerContext &C) const { + if (!HandlerFn.matches(Call)) + return; + + const MemRegion *ArgReg = Call.getArgSVal(0).getAsRegion(); + assert(ArgReg && "expecting a location as the first argument"); + + auto DescriptiveName = ArgReg->getDescriptiveName(/*UseQuotes=*/false); + if (ExplodedNode *Node = C.generateNonFatalErrorNode(C.getState())) { + auto Report = + std::make_unique(Bug, DescriptiveName, Node); + C.emitReport(std::move(Report)); + } + } + +private: + const BugType Bug{this, "DescriptiveNameBug"}; + const CallDescription HandlerFn = {{"reportDescriptiveName"}, 1}; +}; + +void addDescriptiveNameChecker(AnalysisASTConsumer &AnalysisConsumer, + AnalyzerOptions &AnOpts) { + AnOpts.CheckersAndPackages = {{"DescriptiveNameChecker", true}}; + AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) { + Registry.addChecker("DescriptiveNameChecker", + "Desc", "DocsURI"); + }); +} + +bool runChecker(StringRef Code, std::string &Output) { + return runCheckerOnCode(Code.str(), Output, + /*OnlyEmitWarnings=*/true); +} + +TEST(MemRegionDescriptiveNameTest, ConcreteIntElementRegionIndex) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +const unsigned int index = 1; +extern int array[3]; +void top() { + reportDescriptiveName(&array[index]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: array[1]\n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndex) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern unsigned int index; +extern int array[3]; +void top() { + reportDescriptiveName(&array[index]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: array[index]\n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexSymbolValFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int* ptr; +extern int array[3]; +void top() { + reportDescriptiveName(&array[(long)ptr]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexOrigRegionFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int getInt(void); +extern int array[3]; +void top() { + reportDescriptiveName(&array[getInt()]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexDescrNameFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int *ptr; +extern int array[3]; +void top() { + reportDescriptiveName(&array[*ptr]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, + SymbolicElementRegionIndexIncorrectSymbolName) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int x, y; +extern int array[3]; +void top() { + y = x; + reportDescriptiveName(&array[y]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + // FIXME: Should return array[y], but returns array[x] (OriginRegion). + EXPECT_EQ(Output, "DescriptiveNameChecker: array[x]\n"); +} + +} // namespace diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn index 01c2b6ced336..9230ac79a48a 100644 --- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn @@ -19,6 +19,7 @@ unittest("StaticAnalysisTests") { "CallEventTest.cpp", "ConflictingEvalCallsTest.cpp", "FalsePositiveRefutationBRVisitorTest.cpp", + "MemRegionDescriptiveNameTest.cpp", "NoStateChangeFuncVisitorTest.cpp", "ParamRegionTest.cpp", "RangeSetTest.cpp", -- GitLab From aa571a1e8f03faa311fb135ea4b24b9414db1b74 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 21 Mar 2024 17:34:43 +0000 Subject: [PATCH 082/933] [gn build] Port 797336b1278c --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 5ba91fcec83a..12d875cf40c9 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -104,7 +104,6 @@ static_library("MCTargetDesc") { "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", "AMDGPUMCExpr.cpp", - "AMDGPUMCKernelDescriptor.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From e84a985cbf4b26c10812e9bd339db9cfd037f581 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Thu, 21 Mar 2024 13:50:40 -0400 Subject: [PATCH 083/933] [Flang] Support for NULL() and procedure in structure constructor for procedure pointer component. (#85991) This PR fixes a subset of procedure pointer component initialization in structure constructor. It covers 1. NULL() 2. procedure For example: ``` MODULE M TYPE :: DT !PROCEDURE(Fun), POINTER, NOPASS :: pp1 PROCEDURE(Fun), POINTER :: pp1 END TYPE CONTAINS INTEGER FUNCTION Fun(Arg) class(dt) :: arg END FUNCTION END MODULE PROGRAM MAIN USE M IMPLICIT NONE TYPE (DT), PARAMETER :: v1 = DT(NULL()) TYPE (DT) :: v2 v2 = DT(fun) END ``` Passing a procedure pointer itself or reference to a function that returns a procedure pointer is TODO. --- flang/lib/Lower/ConvertConstant.cpp | 22 +++++++-- ...ointer-component-structure-constructor.f90 | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp index 336944d35b7e..ed389bbe4ae5 100644 --- a/flang/lib/Lower/ConvertConstant.cpp +++ b/flang/lib/Lower/ConvertConstant.cpp @@ -14,9 +14,12 @@ #include "flang/Evaluate/expression.h" #include "flang/Lower/AbstractConverter.h" #include "flang/Lower/BuiltinModules.h" +#include "flang/Lower/ConvertExprToHLFIR.h" #include "flang/Lower/ConvertType.h" #include "flang/Lower/ConvertVariable.h" #include "flang/Lower/Mangler.h" +#include "flang/Lower/StatementContext.h" +#include "flang/Lower/SymbolMap.h" #include "flang/Optimizer/Builder/Complex.h" #include "flang/Optimizer/Builder/MutableBox.h" #include "flang/Optimizer/Builder/Todo.h" @@ -380,10 +383,21 @@ static mlir::Value genStructureComponentInit( } if (Fortran::semantics::IsPointer(sym)) { - if (Fortran::semantics::IsProcedure(sym)) - TODO(loc, "procedure pointer component initial value"); - mlir::Value initialTarget = - Fortran::lower::genInitialDataTarget(converter, loc, componentTy, expr); + mlir::Value initialTarget; + if (Fortran::semantics::IsProcedure(sym)) { + if (Fortran::evaluate::UnwrapExpr(expr)) + initialTarget = + fir::factory::createNullBoxProc(builder, loc, componentTy); + else { + Fortran::lower::SymMap globalOpSymMap; + Fortran::lower::StatementContext stmtCtx; + auto box{getBase(Fortran::lower::convertExprToAddress( + loc, converter, expr, globalOpSymMap, stmtCtx))}; + initialTarget = builder.createConvert(loc, componentTy, box); + } + } else + initialTarget = Fortran::lower::genInitialDataTarget(converter, loc, + componentTy, expr); res = builder.create( loc, recTy, res, initialTarget, builder.getArrayAttr(field.getAttributes())); diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 new file mode 100644 index 000000000000..f41c832ee5ec --- /dev/null +++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 @@ -0,0 +1,48 @@ +! Test passing +! 1. NULL(), +! 2. procedure, +! 3. procedure pointer, (pending) +! 4. reference to a function that returns a procedure pointer (pending) +! to a derived type structure constructor. +! RUN: bbc -emit-hlfir -o - %s | FileCheck %s + + MODULE M + TYPE :: DT + PROCEDURE(Fun), POINTER, NOPASS :: pp1 + END TYPE + + CONTAINS + + INTEGER FUNCTION Fun(Arg) + INTEGER :: Arg + Fun = Arg + END FUNCTION + + END MODULE + + PROGRAM MAIN + USE M + IMPLICIT NONE + TYPE (DT), PARAMETER :: v1 = DT(NULL()) + TYPE (DT) :: v2 + v2 = DT(fun) + END + +! CDHECK-LABEL: fir.global internal @_QFECv1 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> { +! CHECK: %[[VAL_0:.*]] = fir.undefined !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_1:.*]] = fir.field_index pp1, !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_2:.*]] = fir.zero_bits (!fir.ref) -> i32 +! CHECK: %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : ((!fir.ref) -> i32) -> !fir.boxproc<(!fir.ref) -> i32> +! CHECK: %[[VAL_4:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_3]], ["pp1", !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>] : (!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>, !fir.boxproc<(!fir.ref) -> i32>) -> !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: fir.has_value %[[VAL_4]] : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: } + +! CHECK-LABEL: fir.global internal @_QQro._QMmTdt.0 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> { +! CHECK: %[[VAL_0:.*]] = fir.undefined !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_1:.*]] = fir.field_index pp1, !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_2:.*]] = fir.address_of(@_QMmPfun) : (!fir.ref) -> i32 +! CHECK: %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : ((!fir.ref) -> i32) -> !fir.boxproc<() -> ()> +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<(!fir.ref) -> i32> +! CHECK: %[[VAL_5:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_4]], ["pp1", !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>] : (!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>, !fir.boxproc<(!fir.ref) -> i32>) -> !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: fir.has_value %[[VAL_5]] : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: } -- GitLab From 3218570620a320573dfd3315da8277600a010bc1 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 17:52:53 +0000 Subject: [PATCH 084/933] [X86] Add shuffle test case for Issue #86068 --- llvm/test/CodeGen/X86/oddshuffles.ll | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll index 5da18ee6ad7c..01056a8b2c24 100644 --- a/llvm/test/CodeGen/X86/oddshuffles.ll +++ b/llvm/test/CodeGen/X86/oddshuffles.ll @@ -2369,6 +2369,31 @@ define void @PR41097() { ret void } +; FIXME - should use INSERTPS +define <2 x float> @PR86068(<2 x float> %0, <2 x float> %1) { +; SSE2-LABEL: PR86068: +; SSE2: # %bb.0: # %entry +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1] +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[1,1] +; SSE2-NEXT: retq +; +; SSE42-LABEL: PR86068: +; SSE42: # %bb.0: # %entry +; SSE42-NEXT: movshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE42-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3] +; SSE42-NEXT: retq +; +; AVX-LABEL: PR86068: +; AVX: # %bb.0: # %entry +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3] +; AVX-NEXT: retq +entry: + %3 = shufflevector <2 x float> %1, <2 x float> poison, <2 x i32> + %4 = shufflevector <2 x float> %3, <2 x float> %0, <2 x i32> + ret <2 x float> %4 +} + define void @D107009(ptr %input, ptr %output) { ; SSE-LABEL: D107009: ; SSE: # %bb.0: -- GitLab From aa7f8200ac29823e6c662d1f41763c2509514161 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 21:33:02 -0500 Subject: [PATCH 085/933] [IR] Add helpers for `NUWAddLike` and `NSWAddLike` to also match `or disjoint`; NFC `or disjoint` implies `add nuw nsw`: https://alive2.llvm.org/ce/z/VABhDA --- llvm/include/llvm/IR/PatternMatch.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 382009d9df78..04feb1fd17fd 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1319,6 +1319,26 @@ m_AddLike(const LHS &L, const RHS &R) { return m_CombineOr(m_Add(L, R), m_DisjointOr(L, R)); } +/// Match either "add nsw" or "or disjoint" +template +inline match_combine_or< + OverflowingBinaryOp_match, + DisjointOr_match> +m_NSWAddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_NSWAdd(L, R), m_DisjointOr(L, R)); +} + +/// Match either "add nuw" or "or disjoint" +template +inline match_combine_or< + OverflowingBinaryOp_match, + DisjointOr_match> +m_NUWAddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_NUWAdd(L, R), m_DisjointOr(L, R)); +} + //===----------------------------------------------------------------------===// // Class that matches a group of binary opcodes. // -- GitLab From ac13e5c0f698c1e3f759e622d2d3cf36ab3b77f3 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 22:09:23 -0500 Subject: [PATCH 086/933] [InstCombine] Add tests for integrating `N{U,S}WAddLike`; NFC --- llvm/test/Transforms/InstCombine/add.ll | 79 +++++++++++++++++++ llvm/test/Transforms/InstCombine/div.ll | 26 ++++++ .../InstCombine/sadd-with-overflow.ll | 33 ++++++++ llvm/test/Transforms/InstCombine/shift-add.ll | 33 ++++++++ .../InstCombine/uadd-with-overflow.ll | 24 ++++++ 5 files changed, 195 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll index 522dcf8db27f..846da7f760b1 100644 --- a/llvm/test/Transforms/InstCombine/add.ll +++ b/llvm/test/Transforms/InstCombine/add.ll @@ -3986,5 +3986,84 @@ define i32 @add_reduce_sqr_sum_varC_invalid2(i32 %a, i32 %b) { ret i32 %ab2 } +define i32 @fold_sext_addition_or_disjoint(i8 %x) { +; CHECK-LABEL: @fold_sext_addition_or_disjoint( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 12 + %se = sext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_sext_addition_fail(i8 %x) { +; CHECK-LABEL: @fold_sext_addition_fail( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 12 + %se = sext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_or_disjoint(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_or_disjoint( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 12 + %se = zext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_or_disjoint2(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_or_disjoint2( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 18 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 18 + %se = zext i8 %xx to i32 + %r = add i32 %se, -14 + ret i32 %r +} + +define i32 @fold_zext_addition_fail(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_fail( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 12 + %se = zext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_fail2(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_fail2( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 18 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 18 + %se = zext i8 %xx to i32 + %r = add i32 %se, -14 + ret i32 %r +} + + declare void @llvm.assume(i1) declare void @fake_func(i32) diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll index 1309dee817cf..a4d604b329c3 100644 --- a/llvm/test/Transforms/InstCombine/div.ll +++ b/llvm/test/Transforms/InstCombine/div.ll @@ -1810,3 +1810,29 @@ define i6 @udiv_distribute_mul_nsw_add_nuw(i6 %x) { %div = udiv i6 %add, 3 ret i6 %div } + +define i32 @fold_disjoint_or_over_sdiv(i32 %x) { +; CHECK-LABEL: @fold_disjoint_or_over_sdiv( +; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[X:%.*]], 9 +; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 +; CHECK-NEXT: [[R:%.*]] = sdiv i32 [[OR]], 9 +; CHECK-NEXT: ret i32 [[R]] +; + %mul = mul nsw i32 %x, 9 + %or = or disjoint i32 %mul, 81 + %r = sdiv i32 %or, 9 + ret i32 %r +} + +define i32 @fold_disjoint_or_over_udiv(i32 %x) { +; CHECK-LABEL: @fold_disjoint_or_over_udiv( +; CHECK-NEXT: [[MUL:%.*]] = mul nuw i32 [[X:%.*]], 9 +; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 +; CHECK-NEXT: [[R:%.*]] = udiv i32 [[OR]], 9 +; CHECK-NEXT: ret i32 [[R]] +; + %mul = mul nuw i32 %x, 9 + %or = or disjoint i32 %mul, 81 + %r = udiv i32 %or, 9 + ret i32 %r +} diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll index 4b37ccbe3370..904dd480caaf 100644 --- a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll @@ -122,3 +122,36 @@ define { i32, i1 } @fold_sub_simple(i32 %x) { %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 30) ret { i32, i1 } %b } + +define { i32, i1 } @fold_with_distjoin_or(i32 %x) { +; CHECK-LABEL: @fold_with_distjoin_or( +; CHECK-NEXT: [[B:%.*]] = add i32 [[X:%.*]], 6 +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 poison, i1 false }, i32 [[B]], 0 +; CHECK-NEXT: ret { i32, i1 } [[TMP1]] +; + %a = or disjoint i32 %x, 13 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 -7) + ret { i32, i1 } %b +} + +define { i32, i1 } @fold_with_disjoint_or2(i32 %x) { +; CHECK-LABEL: @fold_with_disjoint_or2( +; CHECK-NEXT: [[A:%.*]] = or disjoint i32 [[X:%.*]], 100 +; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: ret { i32, i1 } [[B]] +; + %a = or disjoint i32 %x, 100 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 27) + ret { i32, i1 } %b +} + +define { i32, i1 } @fold_with_or_fail(i32 %x) { +; CHECK-LABEL: @fold_with_or_fail( +; CHECK-NEXT: [[A:%.*]] = or i32 [[X:%.*]], 100 +; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: ret { i32, i1 } [[B]] +; + %a = or i32 %x, 100 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 27) + ret { i32, i1 } %b +} diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 1b2567505993..2be6d9b29d52 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -775,3 +775,36 @@ define <3 x i32> @add3_i96(<3 x i32> %0, <3 x i32> %1) { %25 = insertelement <3 x i32> %24, i32 %20, i32 2 ret <3 x i32> %25 } + +define i8 @shl_fold_or_disjoint_cnt(i8 %x) { +; CHECK-LABEL: @shl_fold_or_disjoint_cnt( +; CHECK-NEXT: [[A:%.*]] = or disjoint i8 [[X:%.*]], 3 +; CHECK-NEXT: [[R:%.*]] = shl i8 2, [[A]] +; CHECK-NEXT: ret i8 [[R]] +; + %a = or disjoint i8 %x, 3 + %r = shl i8 2, %a + ret i8 %r +} + +define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { +; CHECK-LABEL: @ashr_fold_or_disjoint_cnt( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %a = or disjoint <2 x i8> %x, + %r = ashr <2 x i8> , %a + ret <2 x i8> %r +} + +define <2 x i8> @lshr_fold_or_disjoint_cnt_out_of_bounds(<2 x i8> %x) { +; CHECK-LABEL: @lshr_fold_or_disjoint_cnt_out_of_bounds( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %a = or disjoint <2 x i8> %x, + %r = lshr <2 x i8> , %a + ret <2 x i8> %r +} diff --git a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll index 28d309baaa41..b1819e6c3e2b 100644 --- a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll @@ -124,3 +124,27 @@ define { i32, i1 } @no_fold_wrapped_add(i32 %x) { %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 30, i32 %a) ret { i32, i1 } %b } + + +define { <2 x i32>, <2 x i1> } @fold_simple_splat_with_disjoint_or_constant(<2 x i32> %x) { +; CHECK-LABEL: @fold_simple_splat_with_disjoint_or_constant( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i32> [[X:%.*]], +; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] +; + %a = or disjoint <2 x i32> %x, + %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> ) + ret { <2 x i32>, <2 x i1> } %b +} + + +define { <2 x i32>, <2 x i1> } @fold_simple_splat_constant_with_or_fail(<2 x i32> %x) { +; CHECK-LABEL: @fold_simple_splat_constant_with_or_fail( +; CHECK-NEXT: [[A:%.*]] = or <2 x i32> [[X:%.*]], +; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] +; + %a = or <2 x i32> %x, + %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> ) + ret { <2 x i32>, <2 x i1> } %b +} -- GitLab From b3ee127e7dead25fa284b26d2c5a067671d0e896 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 22:09:33 -0500 Subject: [PATCH 087/933] [InstCombine] integrate `N{U,S}WAddLike` into existing folds Just went a quick replacement of `N{U,S}WAdd` with the `Like` variant that old matches `or disjoint` Closes #86082 --- .../Transforms/InstCombine/InstCombineAddSub.cpp | 8 +++++--- .../Transforms/InstCombine/InstCombineCalls.cpp | 5 +++-- .../InstCombine/InstCombineMulDivRem.cpp | 8 ++++---- .../Transforms/InstCombine/InstCombineShifts.cpp | 2 +- llvm/test/Transforms/InstCombine/add.ll | 15 ++++++--------- llvm/test/Transforms/InstCombine/div.ll | 8 ++------ .../Transforms/InstCombine/sadd-with-overflow.ll | 3 +-- llvm/test/Transforms/InstCombine/shift-add.ll | 10 +++------- .../Transforms/InstCombine/uadd-with-overflow.ll | 3 +-- 9 files changed, 26 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index aaf7184a5562..a978e9a643f5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -819,7 +819,7 @@ static Instruction *foldNoWrapAdd(BinaryOperator &Add, Value *X; const APInt *C1, *C2; if (match(Op1, m_APInt(C1)) && - match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_APInt(C2))))) && + match(Op0, m_OneUse(m_ZExt(m_NUWAddLike(m_Value(X), m_APInt(C2))))) && C1->isNegative() && C1->sge(-C2->sext(C1->getBitWidth()))) { Constant *NewC = ConstantInt::get(X->getType(), *C2 + C1->trunc(C2->getBitWidth())); @@ -829,14 +829,16 @@ static Instruction *foldNoWrapAdd(BinaryOperator &Add, // More general combining of constants in the wide type. // (sext (X +nsw NarrowC)) + C --> (sext X) + (sext(NarrowC) + C) Constant *NarrowC; - if (match(Op0, m_OneUse(m_SExt(m_NSWAdd(m_Value(X), m_Constant(NarrowC)))))) { + if (match(Op0, + m_OneUse(m_SExt(m_NSWAddLike(m_Value(X), m_Constant(NarrowC)))))) { Value *WideC = Builder.CreateSExt(NarrowC, Ty); Value *NewC = Builder.CreateAdd(WideC, Op1C); Value *WideX = Builder.CreateSExt(X, Ty); return BinaryOperator::CreateAdd(WideX, NewC); } // (zext (X +nuw NarrowC)) + C --> (zext X) + (zext(NarrowC) + C) - if (match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_Constant(NarrowC)))))) { + if (match(Op0, + m_OneUse(m_ZExt(m_NUWAddLike(m_Value(X), m_Constant(NarrowC)))))) { Value *WideC = Builder.CreateZExt(NarrowC, Ty); Value *NewC = Builder.CreateAdd(WideC, Op1C); Value *WideX = Builder.CreateZExt(X, Ty); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 426b548c074a..526fe5d08059 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2093,8 +2093,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { Value *Arg0 = II->getArgOperand(0); Value *Arg1 = II->getArgOperand(1); bool IsSigned = IID == Intrinsic::sadd_with_overflow; - bool HasNWAdd = IsSigned ? match(Arg0, m_NSWAdd(m_Value(X), m_APInt(C0))) - : match(Arg0, m_NUWAdd(m_Value(X), m_APInt(C0))); + bool HasNWAdd = IsSigned + ? match(Arg0, m_NSWAddLike(m_Value(X), m_APInt(C0))) + : match(Arg0, m_NUWAddLike(m_Value(X), m_APInt(C0))); if (HasNWAdd && match(Arg1, m_APInt(C1))) { bool Overflow; APInt NewC = diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 6e05fd8fb4d6..10a4b1c2060c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -1171,14 +1171,14 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) { // We need a multiple of the divisor for a signed add constant, but // unsigned is fine with any constant pair. if (IsSigned && - match(Op0, m_NSWAdd(m_NSWMul(m_Value(X), m_SpecificInt(*C2)), - m_APInt(C1))) && + match(Op0, m_NSWAddLike(m_NSWMul(m_Value(X), m_SpecificInt(*C2)), + m_APInt(C1))) && isMultiple(*C1, *C2, Quotient, IsSigned)) { return BinaryOperator::CreateNSWAdd(X, ConstantInt::get(Ty, Quotient)); } if (!IsSigned && - match(Op0, m_NUWAdd(m_NUWMul(m_Value(X), m_SpecificInt(*C2)), - m_APInt(C1)))) { + match(Op0, m_NUWAddLike(m_NUWMul(m_Value(X), m_SpecificInt(*C2)), + m_APInt(C1)))) { return BinaryOperator::CreateNUWAdd(X, ConstantInt::get(Ty, C1->udiv(*C2))); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp index eafd2889ec50..95aa2119e2d8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp @@ -437,7 +437,7 @@ Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) { Value *A; Constant *C, *C1; if (match(Op0, m_Constant(C)) && - match(Op1, m_NUWAdd(m_Value(A), m_Constant(C1)))) { + match(Op1, m_NUWAddLike(m_Value(A), m_Constant(C1)))) { Value *NewC = Builder.CreateBinOp(I.getOpcode(), C, C1); BinaryOperator *NewShiftOp = BinaryOperator::Create(I.getOpcode(), NewC, A); if (I.getOpcode() == Instruction::Shl) { diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll index 846da7f760b1..ec3aca26514c 100644 --- a/llvm/test/Transforms/InstCombine/add.ll +++ b/llvm/test/Transforms/InstCombine/add.ll @@ -3988,9 +3988,8 @@ define i32 @add_reduce_sqr_sum_varC_invalid2(i32 %a, i32 %b) { define i32 @fold_sext_addition_or_disjoint(i8 %x) { ; CHECK-LABEL: @fold_sext_addition_or_disjoint( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 -; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX:%.*]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1246 ; CHECK-NEXT: ret i32 [[R]] ; %xx = or disjoint i8 %x, 12 @@ -4014,9 +4013,8 @@ define i32 @fold_sext_addition_fail(i8 %x) { define i32 @fold_zext_addition_or_disjoint(i8 %x) { ; CHECK-LABEL: @fold_zext_addition_or_disjoint( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 -; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX:%.*]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1246 ; CHECK-NEXT: ret i32 [[R]] ; %xx = or disjoint i8 %x, 12 @@ -4027,10 +4025,9 @@ define i32 @fold_zext_addition_or_disjoint(i8 %x) { define i32 @fold_zext_addition_or_disjoint2(i8 %x) { ; CHECK-LABEL: @fold_zext_addition_or_disjoint2( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 18 +; CHECK-NEXT: [[XX:%.*]] = add nuw i8 [[X:%.*]], 4 ; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 -; CHECK-NEXT: ret i32 [[R]] +; CHECK-NEXT: ret i32 [[SE]] ; %xx = or disjoint i8 %x, 18 %se = zext i8 %xx to i32 diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll index a4d604b329c3..e8a25ff44d02 100644 --- a/llvm/test/Transforms/InstCombine/div.ll +++ b/llvm/test/Transforms/InstCombine/div.ll @@ -1813,9 +1813,7 @@ define i6 @udiv_distribute_mul_nsw_add_nuw(i6 %x) { define i32 @fold_disjoint_or_over_sdiv(i32 %x) { ; CHECK-LABEL: @fold_disjoint_or_over_sdiv( -; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[X:%.*]], 9 -; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 -; CHECK-NEXT: [[R:%.*]] = sdiv i32 [[OR]], 9 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[X:%.*]], 9 ; CHECK-NEXT: ret i32 [[R]] ; %mul = mul nsw i32 %x, 9 @@ -1826,9 +1824,7 @@ define i32 @fold_disjoint_or_over_sdiv(i32 %x) { define i32 @fold_disjoint_or_over_udiv(i32 %x) { ; CHECK-LABEL: @fold_disjoint_or_over_udiv( -; CHECK-NEXT: [[MUL:%.*]] = mul nuw i32 [[X:%.*]], 9 -; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 -; CHECK-NEXT: [[R:%.*]] = udiv i32 [[OR]], 9 +; CHECK-NEXT: [[R:%.*]] = add nuw i32 [[X:%.*]], 9 ; CHECK-NEXT: ret i32 [[R]] ; %mul = mul nuw i32 %x, 9 diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll index 904dd480caaf..729ca03ddfd1 100644 --- a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll @@ -136,8 +136,7 @@ define { i32, i1 } @fold_with_distjoin_or(i32 %x) { define { i32, i1 } @fold_with_disjoint_or2(i32 %x) { ; CHECK-LABEL: @fold_with_disjoint_or2( -; CHECK-NEXT: [[A:%.*]] = or disjoint i32 [[X:%.*]], 100 -; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: [[B:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 127) ; CHECK-NEXT: ret { i32, i1 } [[B]] ; %a = or disjoint i32 %x, 100 diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 2be6d9b29d52..aa3a238e0949 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -778,8 +778,7 @@ define <3 x i32> @add3_i96(<3 x i32> %0, <3 x i32> %1) { define i8 @shl_fold_or_disjoint_cnt(i8 %x) { ; CHECK-LABEL: @shl_fold_or_disjoint_cnt( -; CHECK-NEXT: [[A:%.*]] = or disjoint i8 [[X:%.*]], 3 -; CHECK-NEXT: [[R:%.*]] = shl i8 2, [[A]] +; CHECK-NEXT: [[R:%.*]] = shl i8 16, [[X:%.*]] ; CHECK-NEXT: ret i8 [[R]] ; %a = or disjoint i8 %x, 3 @@ -789,8 +788,7 @@ define i8 @shl_fold_or_disjoint_cnt(i8 %x) { define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { ; CHECK-LABEL: @ashr_fold_or_disjoint_cnt( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[X:%.*]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %a = or disjoint <2 x i8> %x, @@ -800,9 +798,7 @@ define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { define <2 x i8> @lshr_fold_or_disjoint_cnt_out_of_bounds(<2 x i8> %x) { ; CHECK-LABEL: @lshr_fold_or_disjoint_cnt_out_of_bounds( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] -; CHECK-NEXT: ret <2 x i8> [[R]] +; CHECK-NEXT: ret <2 x i8> zeroinitializer ; %a = or disjoint <2 x i8> %x, %r = lshr <2 x i8> , %a diff --git a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll index b1819e6c3e2b..fd5d38bb38dd 100644 --- a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll @@ -128,8 +128,7 @@ define { i32, i1 } @no_fold_wrapped_add(i32 %x) { define { <2 x i32>, <2 x i1> } @fold_simple_splat_with_disjoint_or_constant(<2 x i32> %x) { ; CHECK-LABEL: @fold_simple_splat_with_disjoint_or_constant( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: [[B:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> ) ; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] ; %a = or disjoint <2 x i32> %x, -- GitLab From 796efa8cd5800a42eb8362564be64f3d72512a05 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 11:09:40 -0700 Subject: [PATCH 088/933] [Float2Int] Fix pessimization in the MinBW calculation. (#86051) The MinBW was being calculated using the significant bits of the upper and lower bounds. The upper bound is 1 past the last value in the range so I don't think it should be included. Instead use ConstantRange::getMinSignedBits. I'm still not sure if the +1 is needed after the getMinSignedBits call. --- llvm/lib/Transforms/Scalar/Float2Int.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp index ccca8bcc1a56..de8c05d5689f 100644 --- a/llvm/lib/Transforms/Scalar/Float2Int.cpp +++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp @@ -359,9 +359,7 @@ bool Float2IntPass::validateAndTransform() { // The number of bits required is the maximum of the upper and // lower limits, plus one so it can be signed. - unsigned MinBW = std::max(R.getLower().getSignificantBits(), - R.getUpper().getSignificantBits()) + - 1; + unsigned MinBW = R.getMinSignedBits() + 1; LLVM_DEBUG(dbgs() << "F2I: MinBitwidth=" << MinBW << ", R: " << R << "\n"); // If we've run off the realms of the exactly representable integers, -- GitLab From f5ef9bd26d531d104f44f9e5b283bd2f80c024be Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 11:17:52 -0700 Subject: [PATCH 089/933] [memprof] Call SmallVector::reserve (#86055) With one raw memprof file I have, NumPCs averages about 41. Given the default number of inline elements being 8 for SmallVector, we should reserve the storage in advance. --- llvm/lib/ProfileData/RawMemProfReader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp index 0e2b8668bab7..60c37c417aa0 100644 --- a/llvm/lib/ProfileData/RawMemProfReader.cpp +++ b/llvm/lib/ProfileData/RawMemProfReader.cpp @@ -127,6 +127,7 @@ CallStackMap readStackInfo(const char *Ptr) { endian::readNext(Ptr); SmallVector CallStack; + CallStack.reserve(NumPCs); for (uint64_t J = 0; J < NumPCs; J++) { CallStack.push_back( endian::readNext(Ptr)); -- GitLab From 999d4f840777bf8de26d45947192aa0728edc0fb Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Thu, 21 Mar 2024 14:18:43 -0400 Subject: [PATCH 090/933] Split -Wcast-function-type into a separate group (#86131) We want to add -Wcast-function-type to -Wextra (as done in 1de7e6c8cba27296f3fc16d107822ea0ee856759), but we do not want to add -Wcast-function-type-strict in at the same time (https://lab.llvm.org/buildbot/#/builders/57/builds/33601/steps/5/logs/stdio). This moves the existing warning to a new group (-Wcast-function-type-mismatch), puts the new group under the existing -Wcast-function-type warning group, and adds -Wcast-function-type-mismatch to -Wextra. --- clang/docs/ReleaseNotes.rst | 20 ++++++++++++++++++- clang/include/clang/Basic/DiagnosticGroups.td | 7 +++++-- .../clang/Basic/DiagnosticSemaKinds.td | 2 +- .../Sema/warn-cast-function-type-strict.c | 12 +++++------ .../warn-cast-function-type-strict.cpp | 10 +++++----- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 50990140a53a..fd12bb41be47 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -198,7 +198,25 @@ Modified Compiler Flags ``-Wreturn-type``, and moved some of the diagnostics previously controlled by ``-Wreturn-type`` under this new flag. Fixes #GH72116. -- Added ``-Wcast-function-type`` as a warning enabled by ``-Wextra``. #GH76872 +- Added ``-Wcast-function-type-mismatch`` under the ``-Wcast-function-type`` + warning group. Moved the diagnostic previously controlled by + ``-Wcast-function-type`` to the new warning group and added + ``-Wcast-function-type-mismatch`` to ``-Wextra``. #GH76872 + + .. code-block:: c + + int x(long); + typedef int (f2)(void*); + typedef int (f3)(); + + void func(void) { + // Diagnoses under -Wcast-function-type, -Wcast-function-type-mismatch, + // -Wcast-function-type-strict, -Wextra + f2 *b = (f2 *)x; + // Diagnoses under -Wcast-function-type, -Wcast-function-type-strict + f3 *c = (f3 *)x; + } + Removed Compiler Flags ------------------------- diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index bf03d4e8f67e..44035e2fd16f 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -573,7 +573,10 @@ def SelTypeCast : DiagGroup<"cast-of-sel-type">; def FunctionDefInObjCContainer : DiagGroup<"function-def-in-objc-container">; def BadFunctionCast : DiagGroup<"bad-function-cast">; def CastFunctionTypeStrict : DiagGroup<"cast-function-type-strict">; -def CastFunctionType : DiagGroup<"cast-function-type", [CastFunctionTypeStrict]>; +def CastFunctionTypeMismatch : DiagGroup<"cast-function-type-mismatch">; +def CastFunctionType : DiagGroup<"cast-function-type", + [CastFunctionTypeStrict, + CastFunctionTypeMismatch]>; def ObjCPropertyImpl : DiagGroup<"objc-property-implementation">; def ObjCPropertyNoAttribute : DiagGroup<"objc-property-no-attribute">; def ObjCPropertyAssignOnObjectType : DiagGroup<"objc-property-assign-on-object-type">; @@ -1038,7 +1041,7 @@ def Extra : DiagGroup<"extra", [ EmptyInitStatement, StringConcatation, FUseLdPath, - CastFunctionType, + CastFunctionTypeMismatch, ]>; def Most : DiagGroup<"most", [ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 270af5d24611..fc727cef9cd8 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9058,7 +9058,7 @@ def warn_bad_function_cast : Warning< InGroup, DefaultIgnore; def warn_cast_function_type : Warning< "cast %diff{from $ to $ |}0,1converts to incompatible function type">, - InGroup, DefaultIgnore; + InGroup, DefaultIgnore; def warn_cast_function_type_strict : Warning, InGroup, DefaultIgnore; def err_cast_pointer_to_non_pointer_int : Error< diff --git a/clang/test/Sema/warn-cast-function-type-strict.c b/clang/test/Sema/warn-cast-function-type-strict.c index 8c88f275d2b3..b0a70cf324b7 100644 --- a/clang/test/Sema/warn-cast-function-type-strict.c +++ b/clang/test/Sema/warn-cast-function-type-strict.c @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type -verify -// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type-strict -verify +// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type -verify=expected,strict +// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type-strict -verify=expected,strict // RUN: %clang_cc1 %s -fsyntax-only -Wextra -Wno-ignored-qualifiers -verify int t(int array[static 12]); @@ -32,13 +32,13 @@ f10 *j; void foo(void) { a = (f1 *)x; b = (f2 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} */ - c = (f3 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)()') converts to incompatible function type}} */ + c = (f3 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)()') converts to incompatible function type}} */ d = (f4 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)()') converts to incompatible function type}} */ - e = (f5 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)(void)') converts to incompatible function type}} */ + e = (f5 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)(void)') converts to incompatible function type}} */ f = (f6 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function type}} */ - g = (f7 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} */ + g = (f7 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} */ h = (f8 *)t; i = (f9 *)u; // FIXME: return type qualifier should not be included in the function type . Warning should be absent after this issue is fixed. https://github.com/llvm/llvm-project/issues/39494 . - j = (f10 *)v; /* expected-warning {{cast from 'const int (*)(int)' to 'f10 *' (aka 'int (*)(int)') converts to incompatible function type}} */ + j = (f10 *)v; /* strict-warning {{cast from 'const int (*)(int)' to 'f10 *' (aka 'int (*)(int)') converts to incompatible function type}} */ } diff --git a/clang/test/SemaCXX/warn-cast-function-type-strict.cpp b/clang/test/SemaCXX/warn-cast-function-type-strict.cpp index b3164afde5a0..8887b3c4c5d5 100644 --- a/clang/test/SemaCXX/warn-cast-function-type-strict.cpp +++ b/clang/test/SemaCXX/warn-cast-function-type-strict.cpp @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type -verify -// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type-strict -verify +// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type -verify=expected,strict +// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type-strict -verify=expected,strict // RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wextra -verify int x(long); @@ -33,11 +33,11 @@ void foo() { a = (f1 *)x; b = (f2 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} b = reinterpret_cast(x); // expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} - c = (f3 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)(...)') converts to incompatible function type}} + c = (f3 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)(...)') converts to incompatible function type}} d = (f4 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)(...)') converts to incompatible function type}} - e = (f5 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)()') converts to incompatible function type}} + e = (f5 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)()') converts to incompatible function type}} f = (f6 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function type}} - g = (f7 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} + g = (f7 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} mf p1 = (mf)&S::foo; // expected-warning {{cast from 'void (S::*)(int *)' to 'mf' (aka 'void (S::*)(int)') converts to incompatible function type}} -- GitLab From 3fefeafa49299ef924414bfa1b678e0f656b3618 Mon Sep 17 00:00:00 2001 From: dmaclach Date: Thu, 21 Mar 2024 11:22:35 -0700 Subject: [PATCH 091/933] [OBJC] Allow __attribute__((NSObject)) types be used as lightweight generic specifiers (#84593) As per https://clang.llvm.org/docs/AutomaticReferenceCounting.html#retainable-object-pointers, types with `__attribute__((NSObject))` are retainable, and thus should be eligible to be used as lightweight generic specifiers. Fix for #84592 84592 --- clang/lib/Sema/SemaType.cpp | 5 +++++ clang/test/SemaObjC/attr-objc-NSObject.m | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 clang/test/SemaObjC/attr-objc-NSObject.m diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 7b14323b0674..d7521a5363a3 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -1018,6 +1018,11 @@ static QualType applyObjCTypeArgs(Sema &S, SourceLocation loc, QualType type, return type; } + // Types that have __attribute__((NSObject)) are permitted. + if (typeArg->isObjCNSObjectType()) { + continue; + } + // Dependent types will be checked at instantiation time. if (typeArg->isDependentType()) { continue; diff --git a/clang/test/SemaObjC/attr-objc-NSObject.m b/clang/test/SemaObjC/attr-objc-NSObject.m new file mode 100644 index 000000000000..76a01dcef016 --- /dev/null +++ b/clang/test/SemaObjC/attr-objc-NSObject.m @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -verify -Wno-objc-root-class -fsyntax-only %s + +@interface NSArray<__covariant ObjectType> +- (void)containsObject:(ObjectType)anObject; // expected-note {{passing argument to parameter 'anObject' here}} +- (void)description; +@end + +typedef __attribute__((NSObject)) struct Foo *FooRef; +typedef struct Bar *BarRef; + +void good() { + FooRef object; + NSArray *array; + [array containsObject:object]; + [object description]; +} + +void bad() { + BarRef object; + NSArray *array; // expected-error {{type argument 'BarRef' (aka 'struct Bar *') is neither an Objective-C object nor a block type}} + [array containsObject:object]; // expected-warning {{incompatible pointer types sending 'BarRef' (aka 'struct Bar *') to parameter of type 'id'}} + [object description]; // expected-warning {{receiver type 'BarRef' (aka 'struct Bar *') is not 'id' or interface pointer, consider casting it to 'id'}} +} -- GitLab From 22e7e68a40b8b1aac8b44137685d21ac4b98bd17 Mon Sep 17 00:00:00 2001 From: lorenzo chelini Date: Thu, 21 Mar 2024 19:40:26 +0100 Subject: [PATCH 092/933] [mlir][Affine] Fix unused variable warning (NFC) --- mlir/lib/Dialect/Affine/Utils/Utils.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp index 3dc5539cde3d..8b8ed2578ca5 100644 --- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp +++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp @@ -1792,8 +1792,7 @@ MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType) { MLIRContext *context = memrefType.getContext(); for (unsigned d = 0; d < newRank; ++d) { // Check if this dimension is dynamic. - if (bool isDynDim = - isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims)) { + if (isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims)) { newShape[d] = ShapedType::kDynamic; continue; } -- GitLab From c96b61adc33b9d4ab26e2d0e4bce929b31c48768 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 20:33:05 +0100 Subject: [PATCH 093/933] [libc] Add reverse_iterator comparisons (#86147) https://en.cppreference.com/w/cpp/iterator/reverse_iterator/operator_cmp --- libc/src/__support/CPP/iterator.h | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index c5bfb1912c7b..4d06e181bcf0 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,6 +20,7 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; + using value_type = T; }; template class reverse_iterator { @@ -27,6 +28,8 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; + using value_type = typename iterator_traits::value_type; + using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -38,6 +41,38 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} + LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() == rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() != rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() > rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() >= rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() < rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() <= rhs.base(); + } + + LIBC_INLINE constexpr iterator_type base() const { current; } + LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 3eb58d15b353534fd42a3a0d1eeb7cd33d128b34 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 20:37:04 +0100 Subject: [PATCH 094/933] Revert "[libc] Add reverse_iterator comparisons" (#86186) Reverts llvm/llvm-project#86147 --- libc/src/__support/CPP/iterator.h | 35 ------------------------------- 1 file changed, 35 deletions(-) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index 4d06e181bcf0..c5bfb1912c7b 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,7 +20,6 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; - using value_type = T; }; template class reverse_iterator { @@ -28,8 +27,6 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; - using value_type = typename iterator_traits::value_type; - using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -41,38 +38,6 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} - LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() == rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() != rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() > rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() >= rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() < rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() <= rhs.base(); - } - - LIBC_INLINE constexpr iterator_type base() const { current; } - LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 536cb1fad3ea3edaba8264992c8de2f4b07abc84 Mon Sep 17 00:00:00 2001 From: Michele Scandale Date: Thu, 21 Mar 2024 12:40:18 -0700 Subject: [PATCH 095/933] [InstCombine] Fix for folding select-like `shufflevector` into floating point binary operators. (#85452) Folding a select-like `shufflevector` into a floating point binary operators can only be done if the result is preserved for both case. In particular, if the common operand of the `shufflevector` and the floating point binary operator can be a NaN, then the transformation won't preserve the result value. --- .../InstCombine/InstCombineVectorOps.cpp | 20 ++++++++++++++++--- .../shuffle_select-inseltpoison.ll | 17 +++++++++++++--- .../Transforms/InstCombine/shuffle_select.ll | 17 +++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index 3c4c0f35eb6d..c7f4fb17648c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -2135,7 +2135,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) { return new ShuffleVectorInst(X, Y, NewMask); } -static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { +static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf, + const SimplifyQuery &SQ) { assert(Shuf.isSelect() && "Must have select-equivalent shuffle"); // Are we shuffling together some value and that same value after it has been @@ -2159,6 +2160,19 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { if (!IdC) return nullptr; + Value *X = Op0IsBinop ? Op1 : Op0; + + // Prevent folding in the case the non-binop operand might have NaN values. + // If X can have NaN elements then we have that the floating point math + // operation in the transformed code may not preserve the exact NaN + // bit-pattern -- e.g. `fadd sNaN, 0.0 -> qNaN`. + // This makes the transformation incorrect since the original program would + // have preserved the exact NaN bit-pattern. + // Avoid the folding if X can have NaN elements. + if (Shuf.getType()->getElementType()->isFloatingPointTy() && + !isKnownNeverNaN(X, 0, SQ)) + return nullptr; + // Shuffle identity constants into the lanes that return the original value. // Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4} // Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4} @@ -2175,7 +2189,6 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { // shuf (bop X, C), X, M --> bop X, C' // shuf X, (bop X, C), M --> bop X, C' - Value *X = Op0IsBinop ? Op1 : Op0; Instruction *NewBO = BinaryOperator::Create(BOpcode, X, NewC); NewBO->copyIRFlags(BO); @@ -2241,7 +2254,8 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) { if (Instruction *I = foldSelectShuffleOfSelectShuffle(Shuf)) return I; - if (Instruction *I = foldSelectShuffleWith1Binop(Shuf)) + if (Instruction *I = foldSelectShuffleWith1Binop( + Shuf, getSimplifyQuery().getWithInstruction(&Shuf))) return I; BinaryOperator *B0, *B1; diff --git a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll index 44ec77e471bb..f573ff36d2ce 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll @@ -336,7 +336,18 @@ define <4 x i32> @srem(<4 x i32> %v) { ; Try FP ops/types. -define <4 x float> @fadd(<4 x float> %v) { +define <4 x float> @fadd_maybe_nan(<4 x float> %v) { +; CHECK-LABEL: @fadd_maybe_nan( +; CHECK-NEXT: [[B:%.*]] = fadd <4 x float> [[V:%.*]], +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[V]], <4 x i32> +; CHECK-NEXT: ret <4 x float> [[S]] +; + %b = fadd <4 x float> %v, + %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> + ret <4 x float> %s +} + +define <4 x float> @fadd(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fadd( ; CHECK-NEXT: [[S:%.*]] = fadd <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -359,7 +370,7 @@ define <4 x double> @fsub(<4 x double> %v) { ; Propagate any FMF. -define <4 x float> @fmul(<4 x float> %v) { +define <4 x float> @fmul(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fmul( ; CHECK-NEXT: [[S:%.*]] = fmul nnan ninf <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -380,7 +391,7 @@ define <4 x double> @fdiv_constant_op0(<4 x double> %v) { ret <4 x double> %s } -define <4 x double> @fdiv_constant_op1(<4 x double> %v) { +define <4 x double> @fdiv_constant_op1(<4 x double> nofpclass(nan) %v) { ; CHECK-LABEL: @fdiv_constant_op1( ; CHECK-NEXT: [[S:%.*]] = fdiv reassoc <4 x double> [[V:%.*]], ; CHECK-NEXT: ret <4 x double> [[S]] diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll index a1b0d782b554..efadb5c3c109 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll @@ -336,7 +336,18 @@ define <4 x i32> @srem(<4 x i32> %v) { ; Try FP ops/types. -define <4 x float> @fadd(<4 x float> %v) { +define <4 x float> @fadd_maybe_nan(<4 x float> %v) { +; CHECK-LABEL: @fadd_maybe_nan( +; CHECK-NEXT: [[B:%.*]] = fadd <4 x float> [[V:%.*]], +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[V]], <4 x i32> +; CHECK-NEXT: ret <4 x float> [[S]] +; + %b = fadd <4 x float> %v, + %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> + ret <4 x float> %s +} + +define <4 x float> @fadd(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fadd( ; CHECK-NEXT: [[S:%.*]] = fadd <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -359,7 +370,7 @@ define <4 x double> @fsub(<4 x double> %v) { ; Propagate any FMF. -define <4 x float> @fmul(<4 x float> %v) { +define <4 x float> @fmul(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fmul( ; CHECK-NEXT: [[S:%.*]] = fmul nnan ninf <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -380,7 +391,7 @@ define <4 x double> @fdiv_constant_op0(<4 x double> %v) { ret <4 x double> %s } -define <4 x double> @fdiv_constant_op1(<4 x double> %v) { +define <4 x double> @fdiv_constant_op1(<4 x double> nofpclass(nan) %v) { ; CHECK-LABEL: @fdiv_constant_op1( ; CHECK-NEXT: [[S:%.*]] = fdiv reassoc <4 x double> [[V:%.*]], ; CHECK-NEXT: ret <4 x double> [[S]] -- GitLab From 85ccfb5ed5389a5fb2d58eab12a9266e7ea064ce Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 21 Mar 2024 12:42:55 -0700 Subject: [PATCH 096/933] [HWASan] [NFC] pull logic to get sanitizer ptr out of hwasan (#86024) Also some drive by cleanup removing an unnnecessary argument and a redundant condition. --- .../Transforms/Utils/MemoryTaggingSupport.h | 1 + .../Instrumentation/HWAddressSanitizer.cpp | 22 +++++-------------- .../Transforms/Utils/MemoryTaggingSupport.cpp | 10 +++++++++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h index 0a0e16d2a9e6..158c358a9e46 100644 --- a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h +++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h @@ -84,6 +84,7 @@ bool isLifetimeIntrinsic(Value *V); Value *readRegister(IRBuilder<> &IRB, StringRef Name); Value *getFP(IRBuilder<> &IRB); Value *getPC(const Triple &TargetTriple, IRBuilder<> &IRB); +Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB); } // namespace memtag } // namespace llvm diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 4bdeb6bbab85..3c95610fa3e8 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -363,7 +363,7 @@ private: Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo); Value *getUARTag(IRBuilder<> &IRB); - Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); + Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB); Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag); unsigned retagMask(unsigned AllocaNo); @@ -1219,20 +1219,10 @@ Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) { return UntaggedPtrLong; } -Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { - Module *M = IRB.GetInsertBlock()->getParent()->getParent(); - if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) { - // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER - // in Bionic's libc/private/bionic_tls.h. - Function *ThreadPointerFunc = - Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); - return IRB.CreateConstGEP1_32(Int8Ty, IRB.CreateCall(ThreadPointerFunc), - 0x30); - } - if (ThreadPtrGlobal) - return ThreadPtrGlobal; - - return nullptr; +Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) { + if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) + return memtag::getAndroidSanitizerSlotPtr(IRB); + return ThreadPtrGlobal; } Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) { @@ -1271,7 +1261,7 @@ void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { auto getThreadLongMaybeUntagged = [&]() { if (!SlotPtr) - SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); + SlotPtr = getHwasanThreadSlotPtr(IRB); if (!ThreadLong) ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); // Extract the address field from ThreadLong. Unnecessary on AArch64 with diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp index 8dd1002a6e4a..fd94a120bc66 100644 --- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp +++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp @@ -273,5 +273,15 @@ Value *getFP(IRBuilder<> &IRB) { IRB.getIntPtrTy(M->getDataLayout())); } +Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB) { + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER + // in Bionic's libc/private/bionic_tls.h. + Function *ThreadPointerFunc = + Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); + return IRB.CreateConstGEP1_32(IRB.getInt8Ty(), + IRB.CreateCall(ThreadPointerFunc), 0x30); +} + } // namespace memtag } // namespace llvm -- GitLab From 6d939a6ec69adf284cdbef2034b49fd02ba503fc Mon Sep 17 00:00:00 2001 From: Kevin Frei Date: Thu, 21 Mar 2024 13:09:04 -0700 Subject: [PATCH 097/933] DebugInfoD tests + fixing issues exposed by tests (#85693) Finally getting back to Debuginfod tests: I've migrated the tests in my [earlier PR](https://github.com/llvm/llvm-project/pull/79181) from shell to API (at @JDevlieghere's suggestion) and addressed a couple issues that came about during testing. The tests first test the "normal" situation (no DebugInfoD involvement, just normal debug files sitting around), then the "no debug info" situation (to make sure the test is seeing failure properly), then it tests to validate that when Debuginfod returns the symbols, things work properly. This is duplicated for DWP/split-dwarf scenarios. --------- Co-authored-by: Kevin Frei --- .../Python/lldbsuite/test/make/Makefile.rules | 33 ++- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 38 ++-- .../Plugins/SymbolLocator/CMakeLists.txt | 7 +- .../SymbolVendor/ELF/SymbolVendorELF.cpp | 30 ++- lldb/test/API/debuginfod/Normal/Makefile | 25 +++ .../API/debuginfod/Normal/TestDebuginfod.py | 185 +++++++++++++++++ lldb/test/API/debuginfod/Normal/main.c | 7 + lldb/test/API/debuginfod/SplitDWARF/Makefile | 28 +++ .../SplitDWARF/TestDebuginfodDWP.py | 194 ++++++++++++++++++ lldb/test/API/debuginfod/SplitDWARF/main.c | 7 + 10 files changed, 537 insertions(+), 17 deletions(-) create mode 100644 lldb/test/API/debuginfod/Normal/Makefile create mode 100644 lldb/test/API/debuginfod/Normal/TestDebuginfod.py create mode 100644 lldb/test/API/debuginfod/Normal/main.c create mode 100644 lldb/test/API/debuginfod/SplitDWARF/Makefile create mode 100644 lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py create mode 100644 lldb/test/API/debuginfod/SplitDWARF/main.c diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules index bfd249ccd43f..75efcde1f040 100644 --- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules +++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules @@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../ # # GNUWin32 uname gives "windows32" or "server version windows32" while # some versions of MSYS uname return "MSYS_NT*", but most environments -# standardize on "Windows_NT", so we'll make it consistent here. +# standardize on "Windows_NT", so we'll make it consistent here. # When running tests from Visual Studio, the environment variable isn't # inherited all the way down to the process spawned for make. #---------------------------------------------------------------------- @@ -210,6 +210,12 @@ else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" DSYM = $(EXE).debug endif + + ifeq "$(MAKE_DWP)" "YES" + MAKE_DWO := YES + DWP_NAME = $(EXE).dwp + DYLIB_DWP_NAME = $(DYLIB_NAME).dwp + endif endif LIMIT_DEBUG_INFO_FLAGS = @@ -357,6 +363,7 @@ ifneq "$(OS)" "Darwin" OBJCOPY ?= $(call replace_cc_with,objcopy) ARCHIVER ?= $(call replace_cc_with,ar) + DWP ?= $(call replace_cc_with,dwp) override AR = $(ARCHIVER) endif @@ -527,6 +534,10 @@ ifneq "$(CXX)" "" endif endif +ifeq "$(GEN_GNU_BUILD_ID)" "YES" + LDFLAGS += -Wl,--build-id +endif + #---------------------------------------------------------------------- # DYLIB_ONLY variable can be used to skip the building of a.out. # See the sections below regarding dSYM file as well as the building of @@ -565,11 +576,25 @@ else endif else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" +ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" + cp "$(EXE)" "$(EXE).full" +endif $(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)" endif +ifeq "$(MAKE_DWP)" "YES" + $(DWP) -o "$(DWP_NAME)" $(DWOS) +endif endif + +#---------------------------------------------------------------------- +# Support emitting the content of the GNU build-id into a file +# This needs to be used in conjunction with GEN_GNU_BUILD_ID := YES +#---------------------------------------------------------------------- +$(EXE).uuid : $(EXE) + $(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $< + #---------------------------------------------------------------------- # Make the dylib #---------------------------------------------------------------------- @@ -610,9 +635,15 @@ endif else $(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)" ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" + ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" + cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).full" + endif $(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)" endif +ifeq "$(MAKE_DWP)" "YES" + $(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS) +endif endif #---------------------------------------------------------------------- diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 5f67658f86ea..08ce7b82b0c1 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -4377,26 +4377,38 @@ const std::shared_ptr &SymbolFileDWARF::GetDwpSymbolFile() { FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); ModuleSpec module_spec; module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec(); + FileSpec dwp_filespec; for (const auto &symfile : symfiles.files()) { module_spec.GetSymbolFileSpec() = FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle()); LLDB_LOG(log, "Searching for DWP using: \"{0}\"", module_spec.GetSymbolFileSpec()); - FileSpec dwp_filespec = + dwp_filespec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); if (FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - GetObjectFile()->GetModule(), &dwp_filespec, 0, - FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, - dwp_file_data_offset); - if (dwp_obj_file) { - m_dwp_symfile = std::make_shared( - *this, dwp_obj_file, DIERef::k_file_index_mask); - break; - } + break; + } + } + if (!FileSystem::Instance().Exists(dwp_filespec)) { + LLDB_LOG(log, "No DWP file found locally"); + // Fill in the UUID for the module we're trying to match for, so we can + // find the correct DWP file, as the Debuginfod plugin uses *only* this + // data to correctly match the DWP file with the binary. + module_spec.GetUUID() = m_objfile_sp->GetUUID(); + dwp_filespec = + PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); + } + if (FileSystem::Instance().Exists(dwp_filespec)) { + LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + GetObjectFile()->GetModule(), &dwp_filespec, 0, + FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, + dwp_file_data_offset); + if (dwp_obj_file) { + m_dwp_symfile = std::make_shared( + *this, dwp_obj_file, DIERef::k_file_index_mask); } } if (!m_dwp_symfile) { diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt index ca969626f4ff..3367022639ab 100644 --- a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt +++ b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt @@ -1,5 +1,10 @@ +# Order matters here: the first symbol locator prevents further searching. +# For DWARF binaries that are both stripped and split, the Default plugin +# will return the stripped binary when asked for the ObjectFile, which then +# prevents an unstripped binary from being requested from the Debuginfod +# provider. +add_subdirectory(Debuginfod) add_subdirectory(Default) if (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_subdirectory(DebugSymbols) endif() -add_subdirectory(Debuginfod) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index b5fe35d71032..91b8b4a979e0 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -44,6 +44,25 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() { "executables."; } +// If this is needed elsewhere, it can be exported/moved. +static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, + const FileSpec &file_spec) { + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + // Try to create an ObjectFile from the file_spec. + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), + dwp_file_data_sp, dwp_file_data_offset); + if (!ObjectFileELF::classof(dwp_obj_file.get())) + return false; + // The presence of a debug_cu_index section is the key identifying feature of + // a DWP file. Make sure we don't fill in the section list on dwp_obj_file + // (by calling GetSectionList(false)) as this is invoked before we may have + // all the symbol files collected and available. + return dwp_obj_file && dwp_obj_file->GetSectionList(false)->FindSectionByType( + eSectionTypeDWARFDebugCuIndex, false); +} + // CreateInstance // // Platforms can register a callback to use when creating symbol vendors to @@ -87,8 +106,15 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp, FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); FileSpec dsym_fspec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - if (!dsym_fspec) - return nullptr; + if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) { + // If we have a stripped binary or if we got a DWP file, we should prefer + // symbols in the executable acquired through a plugin. + ModuleSpec unstripped_spec = + PluginManager::LocateExecutableObjectFile(module_spec); + if (!unstripped_spec) + return nullptr; + dsym_fspec = unstripped_spec.GetFileSpec(); + } DataBufferSP dsym_file_data_sp; lldb::offset_t dsym_file_data_offset = 0; diff --git a/lldb/test/API/debuginfod/Normal/Makefile b/lldb/test/API/debuginfod/Normal/Makefile new file mode 100644 index 000000000000..bd2fa623df47 --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/Makefile @@ -0,0 +1,25 @@ +C_SOURCES := main.c + +# For normal (non DWP) Debuginfod tests, we need: + +# * The "full" binary: a.out.debug +# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and +# SPLIT_DEBUG_SYMBOLS set to YES + +# * The stripped binary (a.out) +# Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES + +# * The 'only-keep-debug' binary (a.out.dbg) +# Produced below + +# * The .uuid file (for a little easier testing code) +# Produced below + +# Don't strip the debug info from a.out: +SPLIT_DEBUG_SYMBOLS := YES +SAVE_FULL_DEBUG_BINARY := YES +GEN_GNU_BUILD_ID := YES + +all: a.out.uuid a.out + +include Makefile.rules diff --git a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py new file mode 100644 index 000000000000..eb5efe83c17a --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py @@ -0,0 +1,185 @@ +import os +import shutil +import tempfile +import struct + +import lldb +from lldbsuite.test.decorators import * +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.lldbtest import * + + +def getUUID(aoutuuid): + """ + Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped + to a file already, as part of the build. + """ + with open(aoutuuid, "rb") as f: + data = f.read(36) + if len(data) != 36: + return None + header = struct.unpack_from("<4I", data) + if len(header) != 4: + return None + # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': + if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: + return None + return data[16:].hex() + + +""" +Test support for the DebugInfoD network symbol acquisition protocol. +This one is for simple / no split-dwarf scenarios. + +For no-split-dwarf scenarios, there are 2 variations: +1 - A stripped binary with it's corresponding unstripped binary: +2 - A stripped binary with a corresponding --only-keep-debug symbols file +""" + + +@skipUnlessPlatform(["linux", "freebsd"]) +class DebugInfodTests(TestBase): + # No need to try every flavor of debug inf. + NO_DEBUG_INFO_TESTCASE = True + + def test_normal_no_symbols(self): + """ + Validate behavior with no symbols or symbol locator. + ('baseline negative' behavior) + """ + test_root = self.config_test(["a.out"]) + self.try_breakpoint(False) + + def test_normal_default(self): + """ + Validate behavior with symbols, but no symbol locator. + ('baseline positive' behavior) + """ + test_root = self.config_test(["a.out", "a.out.debug"]) + self.try_breakpoint(True) + + def test_debuginfod_symbols(self): + """ + Test behavior with the full binary available from Debuginfod as + 'debuginfo' from the plug-in. + """ + test_root = self.config_test(["a.out"], "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_executable(self): + """ + Test behavior with the full binary available from Debuginfod as + 'executable' from the plug-in. + """ + test_root = self.config_test(["a.out"], None, "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_okd_symbols(self): + """ + Test behavior with the 'only-keep-debug' symbols available from Debuginfod. + """ + test_root = self.config_test(["a.out"], "a.out.debug") + self.try_breakpoint(True) + + def try_breakpoint(self, should_have_loc): + """ + This function creates a target from self.aout, sets a function-name + breakpoint, and checks to see if we have a file/line location, + as a way to validate that the symbols have been loaded. + should_have_loc specifies if we're testing that symbols have or + haven't been loaded. + """ + target = self.dbg.CreateTarget(self.aout) + self.assertTrue(target and target.IsValid(), "Target is valid") + + bp = target.BreakpointCreateByName("func") + self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") + self.assertEqual(bp.GetNumLocations(), 1) + + loc = bp.GetLocationAtIndex(0) + self.assertTrue(loc and loc.IsValid(), "Location is valid") + addr = loc.GetAddress() + self.assertTrue(addr and addr.IsValid(), "Loc address is valid") + line_entry = addr.GetLineEntry() + self.assertEqual( + should_have_loc, + line_entry != None and line_entry.IsValid(), + "Loc line entry is valid", + ) + if should_have_loc: + self.assertEqual(line_entry.GetLine(), 4) + self.assertEqual( + line_entry.GetFileSpec().GetFilename(), + self.main_source_file.GetFilename(), + ) + self.dbg.DeleteTarget(target) + shutil.rmtree(self.tmp_dir) + + def config_test(self, local_files, debuginfo=None, executable=None): + """ + Set up a test with local_files[] copied to a different location + so that we control which files are, or are not, found in the file system. + Also, create a stand-alone file-system 'hosted' debuginfod server with the + provided debuginfo and executable files (if they exist) + + Make the filesystem look like: + + /tmp//test/[local_files] + + /tmp//cache (for lldb to use as a temp cache) + + /tmp//buildid//executable -> + /tmp//buildid//debuginfo -> + Returns the /tmp/ path + """ + + self.build() + + uuid = getUUID(self.getBuildArtifact("a.out.uuid")) + + self.main_source_file = lldb.SBFileSpec("main.c") + self.tmp_dir = tempfile.mkdtemp() + test_dir = os.path.join(self.tmp_dir, "test") + os.makedirs(test_dir) + + self.aout = "" + # Copy the files used by the test: + for f in local_files: + shutil.copy(self.getBuildArtifact(f), test_dir) + # The first item is the binary to be used for the test + if self.aout == "": + self.aout = os.path.join(test_dir, f) + + use_debuginfod = debuginfo != None or executable != None + + # Populated the 'file://... mocked' Debuginfod server: + if use_debuginfod: + os.makedirs(os.path.join(self.tmp_dir, "cache")) + uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) + os.makedirs(uuid_dir) + if debuginfo: + shutil.copy( + self.getBuildArtifact(debuginfo), + os.path.join(uuid_dir, "debuginfo"), + ) + if executable: + shutil.copy( + self.getBuildArtifact(executable), + os.path.join(uuid_dir, "executable"), + ) + + # Configure LLDB for the test: + self.runCmd( + "settings set symbols.enable-external-lookup %s" + % str(use_debuginfod).lower() + ) + self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") + if use_debuginfod: + self.runCmd( + "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" + % self.tmp_dir + ) + self.runCmd( + "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" + % self.tmp_dir + ) diff --git a/lldb/test/API/debuginfod/Normal/main.c b/lldb/test/API/debuginfod/Normal/main.c new file mode 100644 index 000000000000..4c7184609b45 --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/main.c @@ -0,0 +1,7 @@ +// This is a dump little pair of test files + +int func(int argc, const char *argv[]) { + return (argc + 1) * (argv[argc][0] + 2); +} + +int main(int argc, const char *argv[]) { return func(0, argv); } diff --git a/lldb/test/API/debuginfod/SplitDWARF/Makefile b/lldb/test/API/debuginfod/SplitDWARF/Makefile new file mode 100644 index 000000000000..266d74cf9062 --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/Makefile @@ -0,0 +1,28 @@ +C_SOURCES := main.c + +# For split-dwarf Debuginfod tests, we need: + +# * A .DWP file (a.out.dwp) +# Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES + +# * The "full" binary: it's missing things that live in .dwo's (a.out.debug) +# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and +# SPLIT_DEBUG_SYMBOLS set to YES + +# * The stripped binary (a.out) +# Produced by Makefile.rules + +# * The 'only-keep-debug' binary (a.out.dbg) +# Produced below + +# * The .uuid file (for a little easier testing code) +# Produced here in the rule below + +MAKE_DWP := YES +SPLIT_DEBUG_SYMBOLS := YES +SAVE_FULL_DEBUG_BINARY := YES +GEN_GNU_BUILD_ID := YES + +all: a.out.uuid a.out + +include Makefile.rules diff --git a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py new file mode 100644 index 000000000000..09f91b6f1c6c --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py @@ -0,0 +1,194 @@ +""" +Test support for the DebugInfoD network symbol acquisition protocol. +""" +import os +import shutil +import tempfile +import struct + +import lldb +from lldbsuite.test.decorators import * +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.lldbtest import * + + +def getUUID(aoutuuid): + """ + Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped + to a file already, as part of the build. + """ + with open(aoutuuid, "rb") as f: + data = f.read(36) + if len(data) != 36: + return None + header = struct.unpack_from("<4I", data) + if len(header) != 4: + return None + # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': + if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: + return None + return data[16:].hex() + + +""" +Test support for the DebugInfoD network symbol acquisition protocol. +This file is for split-dwarf (dwp) scenarios. + +1 - A split binary target with it's corresponding DWP file +2 - A stripped, split binary target with an unstripped binary and a DWP file +3 - A stripped, split binary target with an --only-keep-debug symbols file and a DWP file +""" + + +@skipUnlessPlatform(["linux", "freebsd"]) +class DebugInfodDWPTests(TestBase): + # No need to try every flavor of debug inf. + NO_DEBUG_INFO_TESTCASE = True + + def test_normal_stripped(self): + """ + Validate behavior with a stripped binary, no symbols or symbol locator. + """ + self.config_test(["a.out"]) + self.try_breakpoint(False) + + def test_normal_stripped_split_with_dwp(self): + """ + Validate behavior with symbols, but no symbol locator. + """ + self.config_test(["a.out", "a.out.debug", "a.out.dwp"]) + self.try_breakpoint(True) + + def test_normal_stripped_only_dwp(self): + """ + Validate behavior *with* dwp symbols only, but missing other symbols, + but no symbol locator. This shouldn't work: without the other symbols + DWO's appear mostly useless. + """ + self.config_test(["a.out", "a.out.dwp"]) + self.try_breakpoint(False) + + def test_debuginfod_dwp_from_service(self): + """ + Test behavior with the unstripped binary, and DWP from the service. + """ + self.config_test(["a.out.debug"], "a.out.dwp") + self.try_breakpoint(True) + + def test_debuginfod_both_symfiles_from_service(self): + """ + Test behavior with a stripped binary, with the unstripped binary and + dwp symbols from Debuginfod. + """ + self.config_test(["a.out"], "a.out.dwp", "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_both_okd_symfiles_from_service(self): + """ + Test behavior with both the only-keep-debug symbols and the dwp symbols + from Debuginfod. + """ + self.config_test(["a.out"], "a.out.dwp", "a.out.debug") + self.try_breakpoint(True) + + def try_breakpoint(self, should_have_loc): + """ + This function creates a target from self.aout, sets a function-name + breakpoint, and checks to see if we have a file/line location, + as a way to validate that the symbols have been loaded. + should_have_loc specifies if we're testing that symbols have or + haven't been loaded. + """ + target = self.dbg.CreateTarget(self.aout) + self.assertTrue(target and target.IsValid(), "Target is valid") + + bp = target.BreakpointCreateByName("func") + self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") + self.assertEqual(bp.GetNumLocations(), 1) + + loc = bp.GetLocationAtIndex(0) + self.assertTrue(loc and loc.IsValid(), "Location is valid") + addr = loc.GetAddress() + self.assertTrue(addr and addr.IsValid(), "Loc address is valid") + line_entry = addr.GetLineEntry() + self.assertEqual( + should_have_loc, + line_entry != None and line_entry.IsValid(), + "Loc line entry is valid", + ) + if should_have_loc: + self.assertEqual(line_entry.GetLine(), 4) + self.assertEqual( + line_entry.GetFileSpec().GetFilename(), + self.main_source_file.GetFilename(), + ) + self.dbg.DeleteTarget(target) + shutil.rmtree(self.tmp_dir) + + def config_test(self, local_files, debuginfo=None, executable=None): + """ + Set up a test with local_files[] copied to a different location + so that we control which files are, or are not, found in the file system. + Also, create a stand-alone file-system 'hosted' debuginfod server with the + provided debuginfo and executable files (if they exist) + + Make the filesystem look like: + + /tmp//test/[local_files] + + /tmp//cache (for lldb to use as a temp cache) + + /tmp//buildid//executable -> + /tmp//buildid//debuginfo -> + Returns the /tmp/ path + """ + + self.build() + + uuid = getUUID(self.getBuildArtifact("a.out.uuid")) + + self.main_source_file = lldb.SBFileSpec("main.c") + self.tmp_dir = tempfile.mkdtemp() + self.test_dir = os.path.join(self.tmp_dir, "test") + os.makedirs(self.test_dir) + + self.aout = "" + # Copy the files used by the test: + for f in local_files: + shutil.copy(self.getBuildArtifact(f), self.test_dir) + if self.aout == "": + self.aout = os.path.join(self.test_dir, f) + + use_debuginfod = debuginfo != None or executable != None + + # Populated the 'file://... mocked' Debuginfod server: + if use_debuginfod: + os.makedirs(os.path.join(self.tmp_dir, "cache")) + uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) + os.makedirs(uuid_dir) + if debuginfo: + shutil.copy( + self.getBuildArtifact(debuginfo), + os.path.join(uuid_dir, "debuginfo"), + ) + if executable: + shutil.copy( + self.getBuildArtifact(executable), + os.path.join(uuid_dir, "executable"), + ) + os.remove(self.getBuildArtifact("main.dwo")) + # Configure LLDB for the test: + self.runCmd( + "settings set symbols.enable-external-lookup %s" + % str(use_debuginfod).lower() + ) + self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") + if use_debuginfod: + self.runCmd( + "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" + % self.tmp_dir + ) + self.runCmd( + "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" + % self.tmp_dir + ) diff --git a/lldb/test/API/debuginfod/SplitDWARF/main.c b/lldb/test/API/debuginfod/SplitDWARF/main.c new file mode 100644 index 000000000000..4c7184609b45 --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/main.c @@ -0,0 +1,7 @@ +// This is a dump little pair of test files + +int func(int argc, const char *argv[]) { + return (argc + 1) * (argv[argc][0] + 2); +} + +int main(int argc, const char *argv[]) { return func(0, argv); } -- GitLab From 2ab106cbd428984df3dda2f6983d5f956917cb69 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Thu, 21 Mar 2024 15:12:43 -0500 Subject: [PATCH 098/933] [flang][OpenMP] Convert processTODO and remove unused objects (#81627) Remove `ClauseIterator2` and `clauses2` from ClauseProcessor. [Clause representation 5/6] --- flang/lib/Lower/OpenMP/ClauseProcessor.h | 16 +++----- flang/lib/Lower/OpenMP/OpenMP.cpp | 51 +++++++++--------------- 2 files changed, 23 insertions(+), 44 deletions(-) diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h index 1b76eb97e823..8582716e6b9a 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.h +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h @@ -46,13 +46,11 @@ namespace omp { /// methods that relate to clauses that can impact the lowering of that /// construct. class ClauseProcessor { - using ClauseTy = Fortran::parser::OmpClause; - public: ClauseProcessor(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, const Fortran::parser::OmpClauseList &clauses) - : converter(converter), semaCtx(semaCtx), clauses2(clauses), + : converter(converter), semaCtx(semaCtx), clauses(makeList(clauses, semaCtx)) {} // 'Unique' clauses: They can appear at most once in the clause list. @@ -156,7 +154,6 @@ public: private: using ClauseIterator = List::const_iterator; - using ClauseIterator2 = std::list::const_iterator; /// Utility to find a clause within a range in the clause list. template @@ -182,7 +179,6 @@ private: Fortran::lower::AbstractConverter &converter; Fortran::semantics::SemanticsContext &semaCtx; - const Fortran::parser::OmpClauseList &clauses2; List clauses; }; @@ -238,19 +234,17 @@ bool ClauseProcessor::processMotionClauses( template void ClauseProcessor::processTODO(mlir::Location currentLocation, llvm::omp::Directive directive) const { - auto checkUnhandledClause = [&](const auto *x) { + auto checkUnhandledClause = [&](llvm::omp::Clause id, const auto *x) { if (!x) return; TODO(currentLocation, - "Unhandled clause " + - llvm::StringRef(Fortran::parser::ParseTreeDumper::GetNodeName(*x)) - .upper() + + "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() + " in " + llvm::omp::getOpenMPDirectiveName(directive).upper() + " construct"); }; - for (ClauseIterator2 it = clauses2.v.begin(); it != clauses2.v.end(); ++it) - (checkUnhandledClause(std::get_if(&it->u)), ...); + for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it) + (checkUnhandledClause(it->id, std::get_if(&it->u)), ...); } template diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 160ada379a08..d91694c4f639 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -734,9 +734,7 @@ genTaskOp(Fortran::lower::AbstractConverter &converter, cp.processMergeable(mergeableAttr); cp.processPriority(stmtCtx, priorityClauseOperand); cp.processDepend(dependTypeOperands, dependOperands); - cp.processTODO( + cp.processTODO( currentLocation, llvm::omp::Directive::OMPD_task); return genOpWithBody( @@ -762,8 +760,8 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter, llvm::SmallVector allocateOperands, allocatorOperands; ClauseProcessor cp(converter, semaCtx, clauseList); cp.processAllocate(allocatorOperands, allocateOperands); - cp.processTODO( - currentLocation, llvm::omp::Directive::OMPD_taskgroup); + cp.processTODO(currentLocation, + llvm::omp::Directive::OMPD_taskgroup); return genOpWithBody( OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval) .setGenNested(genNested) @@ -1102,16 +1100,11 @@ genTargetOp(Fortran::lower::AbstractConverter &converter, cp.processMap(currentLocation, directive, stmtCtx, mapOperands, &mapSymTypes, &mapSymLocs, &mapSymbols); - cp.processTODO( + cp.processTODO( currentLocation, llvm::omp::Directive::OMPD_target); + // 5.8.1 Implicit Data-Mapping Attribute Rules // The following code follows the implicit data-mapping rules to map all the // symbols used inside the region that have not been explicitly mapped using @@ -1230,8 +1223,8 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter, cp.processDefault(); cp.processNumTeams(stmtCtx, numTeamsClauseOperand); cp.processThreadLimit(stmtCtx, threadLimitClauseOperand); - cp.processTODO( - currentLocation, llvm::omp::Directive::OMPD_teams); + cp.processTODO(currentLocation, + llvm::omp::Directive::OMPD_teams); return genOpWithBody( OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval) @@ -1283,9 +1276,8 @@ static mlir::omp::DeclareTargetDeviceType getDeclareTargetInfo( cp.processEnter(symbolAndClause); cp.processLink(symbolAndClause); cp.processDeviceType(deviceType); - cp.processTODO( - converter.getCurrentLocation(), - llvm::omp::Directive::OMPD_declare_target); + cp.processTODO(converter.getCurrentLocation(), + llvm::omp::Directive::OMPD_declare_target); } return deviceType; @@ -1367,8 +1359,7 @@ genOmpSimpleStandalone(Fortran::lower::AbstractConverter &converter, break; case llvm::omp::Directive::OMPD_taskwait: ClauseProcessor(converter, semaCtx, opClauseList) - .processTODO( + .processTODO( currentLocation, llvm::omp::Directive::OMPD_taskwait); firOpBuilder.create(currentLocation); break; @@ -1550,11 +1541,8 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, cp.processIf(clause::If::DirectiveNameModifier::Simd, ifClauseOperand); cp.processSimdlen(simdlenClauseOperand); cp.processSafelen(safelenClauseOperand); - cp.processTODO(loc, ompDirective); + cp.processTODO(loc, ompDirective); mlir::TypeRange resultType; auto simdLoopOp = firOpBuilder.create( @@ -1607,8 +1595,7 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols, &reductionSymbols); - cp.processTODO(loc, ompDirective); + cp.processTODO(loc, ompDirective); if (ReductionProcessor::doReductionByRef(reductionVars)) byrefOperand = firOpBuilder.getUnitAttr(); @@ -1670,11 +1657,9 @@ static void createSimdWsloop( const Fortran::parser::OmpClauseList &beginClauseList, const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) { ClauseProcessor cp(converter, semaCtx, beginClauseList); - cp.processTODO< - Fortran::parser::OmpClause::Aligned, Fortran::parser::OmpClause::Allocate, - Fortran::parser::OmpClause::Linear, Fortran::parser::OmpClause::Safelen, - Fortran::parser::OmpClause::Simdlen, Fortran::parser::OmpClause::Order>( - loc, ompDirective); + cp.processTODO(loc, + ompDirective); // TODO: Add support for vectorization - add vectorization hints inside loop // body. // OpenMP standard does not specify the length of vector instructions. -- GitLab From 6b1cf0040059c407264d2609403c4fc090673167 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Thu, 21 Mar 2024 14:05:21 -0700 Subject: [PATCH 099/933] [BOLT] Add support for Linux kernel static keys jump table (#86090) Runtime code modification used by static keys is the most ubiquitous self-modifying feature of the Linux kernel. The idea is to to eliminate the condition check and associated conditional jump on a hot path if that condition (based on a boolean value of a static key) does not change often. Whenever they condition changes, the kernel runtime modifies all code paths associated with that key flipping the code between nop and (unconditional) jump. --- bolt/include/bolt/Core/MCPlus.h | 1 + bolt/include/bolt/Core/MCPlusBuilder.h | 17 + bolt/lib/Core/BinaryContext.cpp | 8 +- bolt/lib/Core/BinaryFunction.cpp | 17 +- bolt/lib/Core/MCPlusBuilder.cpp | 22 ++ bolt/lib/Passes/BinaryPasses.cpp | 20 +- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 385 +++++++++++++++++++++++ bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 13 + bolt/test/X86/linux-static-keys.s | 67 ++++ 9 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 bolt/test/X86/linux-static-keys.s diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h index b6a9e73f2347..1d2360c18033 100644 --- a/bolt/include/bolt/Core/MCPlus.h +++ b/bolt/include/bolt/Core/MCPlus.h @@ -73,6 +73,7 @@ public: kOffset, /// Offset in the function. kLabel, /// MCSymbol pointing to this instruction. kSize, /// Size of the instruction. + kDynamicBranch, /// Jit instruction patched at runtime. kGeneric /// First generic annotation. }; diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 96b58f541623..198a8d8bf48f 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -1199,6 +1199,16 @@ public: /// Set instruction size. void setSize(MCInst &Inst, uint32_t Size) const; + /// Check if the branch instruction could be modified at runtime. + bool isDynamicBranch(const MCInst &Inst) const; + + /// Return ID for runtime-modifiable instruction. + std::optional getDynamicBranchID(const MCInst &Inst) const; + + /// Mark instruction as a dynamic branch, i.e. a branch that can be + /// overwritten at runtime. + void setDynamicBranch(MCInst &Inst, uint32_t ID) const; + /// Return MCSymbol that represents a target of this instruction at a given /// operand number \p OpNum. If there's no symbol associated with /// the operand - return nullptr. @@ -1688,6 +1698,13 @@ public: llvm_unreachable("not implemented"); } + /// Create long conditional branch with a target-specific conditional code + /// \p CC. + virtual void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, + unsigned CC, MCContext *Ctx) const { + llvm_unreachable("not implemented"); + } + /// Reverses the branch condition in Inst and update its taken target to TBB. /// /// Returns true on success. diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index b29ebbbfa18c..267f43f65e20 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -1939,7 +1939,13 @@ void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction, OS << Endl; return; } - InstPrinter->printInst(&Instruction, 0, "", *STI, OS); + if (std::optional DynamicID = + MIB->getDynamicBranchID(Instruction)) { + OS << "\tjit\t" << MIB->getTargetSymbol(Instruction)->getName() + << " # ID: " << DynamicID; + } else { + InstPrinter->printInst(&Instruction, 0, "", *STI, OS); + } if (MIB->isCall(Instruction)) { if (MIB->isTailCall(Instruction)) OS << " # TAILCALL "; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index ce4dd29f542b..fdadef9dcd38 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3350,6 +3350,16 @@ void BinaryFunction::fixBranches() { // Eliminate unnecessary conditional branch. if (TSuccessor == FSuccessor) { + // FIXME: at the moment, we cannot safely remove static key branches. + if (MIB->isDynamicBranch(*CondBranch)) { + if (opts::Verbosity) { + BC.outs() + << "BOLT-INFO: unable to remove redundant dynamic branch in " + << *this << '\n'; + } + continue; + } + BB->removeDuplicateConditionalSuccessor(CondBranch); if (TSuccessor != NextBB) BB->addBranchInstruction(TSuccessor); @@ -3358,8 +3368,13 @@ void BinaryFunction::fixBranches() { // Reverse branch condition and swap successors. auto swapSuccessors = [&]() { - if (MIB->isUnsupportedBranch(*CondBranch)) + if (MIB->isUnsupportedBranch(*CondBranch)) { + if (opts::Verbosity) { + BC.outs() << "BOLT-INFO: unable to swap successors in " << *this + << '\n'; + } return false; + } std::swap(TSuccessor, FSuccessor); BB->swapConditionalSuccessors(); auto L = BC.scopeLock(); diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index bd9bd0c45922..5b14ad5cdb88 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -303,6 +303,28 @@ void MCPlusBuilder::setSize(MCInst &Inst, uint32_t Size) const { setAnnotationOpValue(Inst, MCAnnotation::kSize, Size); } +bool MCPlusBuilder::isDynamicBranch(const MCInst &Inst) const { + if (!hasAnnotation(Inst, MCAnnotation::kDynamicBranch)) + return false; + assert(isBranch(Inst) && "Branch expected."); + return true; +} + +std::optional +MCPlusBuilder::getDynamicBranchID(const MCInst &Inst) const { + if (std::optional Value = + getAnnotationOpValue(Inst, MCAnnotation::kDynamicBranch)) { + assert(isBranch(Inst) && "Branch expected."); + return static_cast(*Value); + } + return std::nullopt; +} + +void MCPlusBuilder::setDynamicBranch(MCInst &Inst, uint32_t ID) const { + assert(isBranch(Inst) && "Branch expected."); + setAnnotationOpValue(Inst, MCAnnotation::kDynamicBranch, ID); +} + bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, unsigned Index) const { return (bool)getAnnotationOpValue(Inst, Index); } diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp index bf1c2ddd37dd..c0ba73108f57 100644 --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -107,6 +107,12 @@ static cl::opt cl::desc("print statistics about basic block ordering"), cl::init(0), cl::cat(BoltOptCategory)); +static cl::opt PrintLargeFunctions( + "print-large-functions", + cl::desc("print functions that could not be overwritten due to excessive " + "size"), + cl::init(false), cl::cat(BoltOptCategory)); + static cl::list PrintSortedBy("print-sorted-by", cl::CommaSeparated, cl::desc("print functions sorted by order of dyno stats"), @@ -570,8 +576,12 @@ Error CheckLargeFunctions::runOnFunctions(BinaryContext &BC) { uint64_t HotSize, ColdSize; std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(BF, /*FixBranches=*/false); - if (HotSize > BF.getMaxSize()) + if (HotSize > BF.getMaxSize()) { + if (opts::PrintLargeFunctions) + BC.outs() << "BOLT-INFO: " << BF << " size exceeds allocated space by " + << (HotSize - BF.getMaxSize()) << " bytes\n"; BF.setSimple(false); + } }; ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) { @@ -852,6 +862,10 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { assert(Result && "internal error analyzing conditional branch"); assert(CondBranch && "conditional branch expected"); + // Skip dynamic branches for now. + if (BF.getBinaryContext().MIB->isDynamicBranch(*CondBranch)) + continue; + // It's possible that PredBB is also a successor to BB that may have // been processed by a previous iteration of the SCTC loop, in which // case it may have been marked invalid. We should skip rewriting in @@ -1012,6 +1026,10 @@ uint64_t ShortenInstructions::shortenInstructions(BinaryFunction &Function) { const BinaryContext &BC = Function.getBinaryContext(); for (BinaryBasicBlock &BB : Function) { for (MCInst &Inst : BB) { + // Skip shortening instructions with Size annotation. + if (BC.MIB->getSize(Inst)) + continue; + MCInst OriginalInst; if (opts::Verbosity > 2) OriginalInst = Inst; diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index a2bfd45a64e3..b028a455a6db 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -14,7 +14,9 @@ #include "bolt/Rewrite/MetadataRewriter.h" #include "bolt/Rewrite/MetadataRewriters.h" #include "bolt/Utils/CommandLineOpts.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -65,6 +67,16 @@ static cl::opt DumpStaticCalls("dump-static-calls", cl::init(false), cl::Hidden, cl::cat(BoltCategory)); +static cl::opt + DumpStaticKeys("dump-static-keys", + cl::desc("dump Linux kernel static keys jump table"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + +static cl::opt LongJumpLabels( + "long-jump-labels", + cl::desc("always use long jumps/nops for Linux kernel static keys"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + static cl::opt PrintORC("print-orc", cl::desc("print ORC unwind information for instructions"), @@ -151,6 +163,20 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Number of entries in the input file ORC sections. uint64_t NumORCEntries = 0; + /// Section containing static keys jump table. + ErrorOr StaticKeysJumpSection = std::errc::bad_address; + uint64_t StaticKeysJumpTableAddress = 0; + static constexpr size_t STATIC_KEYS_JUMP_ENTRY_SIZE = 8; + + struct JumpInfoEntry { + bool Likely; + bool InitValue; + }; + SmallVector JumpInfo; + + /// Static key entries that need nop conversion. + DenseSet NopIDs; + /// Section containing static call table. ErrorOr StaticCallSection = std::errc::bad_address; uint64_t StaticCallTableAddress = 0; @@ -235,6 +261,11 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Read .pci_fixup Error readPCIFixupTable(); + /// Handle static keys jump table. + Error readStaticKeysJumpTable(); + Error rewriteStaticKeysJumpTable(); + Error updateStaticKeysJumpTablePostEmit(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -268,6 +299,9 @@ public: if (Error E = readPCIFixupTable()) return E; + if (Error E = readStaticKeysJumpTable()) + return E; + return Error::success(); } @@ -290,12 +324,18 @@ public: if (Error E = rewriteStaticCalls()) return E; + if (Error E = rewriteStaticKeysJumpTable()) + return E; + return Error::success(); } Error postEmitFinalizer() override { updateLKMarkers(); + if (Error E = updateStaticKeysJumpTablePostEmit()) + return E; + return Error::success(); } }; @@ -1343,6 +1383,351 @@ Error LinuxKernelRewriter::readPCIFixupTable() { return Error::success(); } +/// Runtime code modification used by static keys is the most ubiquitous +/// self-modifying feature of the Linux kernel. The idea is to eliminate the +/// condition check and associated conditional jump on a hot path if that +/// condition (based on a boolean value of a static key) does not change often. +/// Whenever the condition changes, the kernel runtime modifies all code paths +/// associated with that key flipping the code between nop and (unconditional) +/// jump. The information about the code is stored in a static key jump table +/// and contains the list of entries of the following type from +/// include/linux/jump_label.h: +// +/// struct jump_entry { +/// s32 code; +/// s32 target; +/// long key; // key may be far away from the core kernel under KASLR +/// }; +/// +/// The list does not have to be stored in any sorted way, but it is sorted at +/// boot time (or module initialization time) first by "key" and then by "code". +/// jump_label_sort_entries() is responsible for sorting the table. +/// +/// The key in jump_entry structure uses lower two bits of the key address +/// (which itself is aligned) to store extra information. We are interested in +/// the lower bit which indicates if the key is likely to be set on the code +/// path associated with this jump_entry. +/// +/// static_key_{enable,disable}() functions modify the code based on key and +/// jump table entries. +/// +/// jump_label_update() updates all code entries for a given key. Batch mode is +/// used for x86. +/// +/// The actual patching happens in text_poke_bp_batch() that overrides the first +/// byte of the sequence with int3 before proceeding with actual code +/// replacement. +Error LinuxKernelRewriter::readStaticKeysJumpTable() { + const BinaryData *StaticKeysJumpTable = + BC.getBinaryDataByName("__start___jump_table"); + if (!StaticKeysJumpTable) + return Error::success(); + + StaticKeysJumpTableAddress = StaticKeysJumpTable->getAddress(); + + const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table"); + if (!Stop) + return createStringError(errc::executable_format_error, + "missing __stop___jump_table symbol"); + + ErrorOr ErrorOrSection = + BC.getSectionForAddress(StaticKeysJumpTableAddress); + if (!ErrorOrSection) + return createStringError(errc::executable_format_error, + "no section matching __start___jump_table"); + + StaticKeysJumpSection = *ErrorOrSection; + if (!StaticKeysJumpSection->containsAddress(Stop->getAddress() - 1)) + return createStringError(errc::executable_format_error, + "__stop___jump_table not in the same section " + "as __start___jump_table"); + + if ((Stop->getAddress() - StaticKeysJumpTableAddress) % + STATIC_KEYS_JUMP_ENTRY_SIZE) + return createStringError(errc::executable_format_error, + "static keys jump table size error"); + + const uint64_t SectionAddress = StaticKeysJumpSection->getAddress(); + DataExtractor DE(StaticKeysJumpSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress); + uint32_t EntryID = 0; + while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) { + const uint64_t JumpAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t TargetAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t KeyAddress = + SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor); + + // Consume the status of the cursor. + if (!Cursor) + return createStringError( + errc::executable_format_error, + "out of bounds while reading static keys jump table: %s", + toString(Cursor.takeError()).c_str()); + + ++EntryID; + + JumpInfo.push_back(JumpInfoEntry()); + JumpInfoEntry &Info = JumpInfo.back(); + Info.Likely = KeyAddress & 1; + + if (opts::DumpStaticKeys) { + BC.outs() << "Static key jump entry: " << EntryID + << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress) + << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) + << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) + << "\n\tIsLikely: " << Info.Likely << '\n'; + } + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(JumpAddress); + if (!BF && opts::Verbosity) { + BC.outs() + << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(JumpAddress) + << " of jump instruction referenced from static keys jump table\n"; + } + + if (!BF || !BC.shouldEmit(*BF)) + continue; + + MCInst *Inst = BF->getInstructionAtOffset(JumpAddress - BF->getAddress()); + if (!Inst) + return createStringError( + errc::executable_format_error, + "no instruction at static keys jump site address 0x%" PRIx64, + JumpAddress); + + if (!BF->containsAddress(TargetAddress)) + return createStringError( + errc::executable_format_error, + "invalid target of static keys jump at 0x%" PRIx64 " : 0x%" PRIx64, + JumpAddress, TargetAddress); + + const bool IsBranch = BC.MIB->isBranch(*Inst); + if (!IsBranch && !BC.MIB->isNoop(*Inst)) + return createStringError(errc::executable_format_error, + "jump or nop expected at address 0x%" PRIx64, + JumpAddress); + + const uint64_t Size = BC.computeInstructionSize(*Inst); + if (Size != 2 && Size != 5) { + return createStringError( + errc::executable_format_error, + "unexpected static keys jump size at address 0x%" PRIx64, + JumpAddress); + } + + MCSymbol *Target = BF->registerBranch(JumpAddress, TargetAddress); + MCInst StaticKeyBranch; + + // Create a conditional branch instruction. The actual conditional code type + // should not matter as long as it's a valid code. The instruction should be + // treated as a conditional branch for control-flow purposes. Before we emit + // the code, it will be converted to a different instruction in + // rewriteStaticKeysJumpTable(). + // + // NB: for older kernels, under LongJumpLabels option, we create long + // conditional branch to guarantee that code size estimation takes + // into account the extra bytes needed for long branch that will be used + // by the kernel patching code. Newer kernels can work with both short + // and long branches. The code for long conditional branch is larger + // than unconditional one, so we are pessimistic in our estimations. + if (opts::LongJumpLabels) + BC.MIB->createLongCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get()); + else + BC.MIB->createCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get()); + BC.MIB->moveAnnotations(std::move(*Inst), StaticKeyBranch); + BC.MIB->setDynamicBranch(StaticKeyBranch, EntryID); + *Inst = StaticKeyBranch; + + // IsBranch = InitialValue ^ LIKELY + // + // 0 0 0 + // 1 0 1 + // 1 1 0 + // 0 1 1 + // + // => InitialValue = IsBranch ^ LIKELY + Info.InitValue = IsBranch ^ Info.Likely; + + // Add annotations to facilitate manual code analysis. + BC.MIB->addAnnotation(*Inst, "Likely", Info.Likely); + BC.MIB->addAnnotation(*Inst, "InitValue", Info.InitValue); + if (!BC.MIB->getSize(*Inst)) + BC.MIB->setSize(*Inst, Size); + + if (opts::LongJumpLabels) + BC.MIB->setSize(*Inst, 5); + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID << " static keys jump entries\n"; + + return Error::success(); +} + +// Pre-emit pass. Convert dynamic branch instructions into jumps that could be +// relaxed. In post-emit pass we will convert those jumps into nops when +// necessary. We do the unconditional conversion into jumps so that the jumps +// can be relaxed and the optimal size of jump/nop instruction is selected. +Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() { + if (!StaticKeysJumpSection) + return Error::success(); + + uint64_t NumShort = 0; + uint64_t NumLong = 0; + for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) { + if (!BC.shouldEmit(BF)) + continue; + + for (BinaryBasicBlock &BB : BF) { + for (MCInst &Inst : BB) { + if (!BC.MIB->isDynamicBranch(Inst)) + continue; + + const uint32_t EntryID = *BC.MIB->getDynamicBranchID(Inst); + MCSymbol *Target = + const_cast(BC.MIB->getTargetSymbol(Inst)); + assert(Target && "Target symbol should be set."); + + const JumpInfoEntry &Info = JumpInfo[EntryID - 1]; + const bool IsBranch = Info.Likely ^ Info.InitValue; + + uint32_t Size = *BC.MIB->getSize(Inst); + if (Size == 2) + ++NumShort; + else if (Size == 5) + ++NumLong; + else + llvm_unreachable("Wrong size for static keys jump instruction."); + + MCInst NewInst; + // Replace the instruction with unconditional jump even if it needs to + // be nop in the binary. + if (opts::LongJumpLabels) { + BC.MIB->createLongUncondBranch(NewInst, Target, BC.Ctx.get()); + } else { + // Newer kernels can handle short and long jumps for static keys. + // Optimistically, emit short jump and check if it gets relaxed into + // a long one during post-emit. Only then convert the jump to a nop. + BC.MIB->createUncondBranch(NewInst, Target, BC.Ctx.get()); + } + + BC.MIB->moveAnnotations(std::move(Inst), NewInst); + Inst = NewInst; + + // Mark the instruction for nop conversion. + if (!IsBranch) + NopIDs.insert(EntryID); + + MCSymbol *Label = + BC.MIB->getOrCreateInstLabel(Inst, "__SK_", BC.Ctx.get()); + + // Create a relocation against the label. + const uint64_t EntryOffset = StaticKeysJumpTableAddress - + StaticKeysJumpSection->getAddress() + + (EntryID - 1) * 16; + StaticKeysJumpSection->addRelocation(EntryOffset, Label, + ELF::R_X86_64_PC32, + /*Addend*/ 0); + StaticKeysJumpSection->addRelocation(EntryOffset + 4, Target, + ELF::R_X86_64_PC32, /*Addend*/ 0); + } + } + } + + BC.outs() << "BOLT-INFO: the input contains " << NumShort << " short and " + << NumLong << " long static keys jumps in optimized functions\n"; + + return Error::success(); +} + +// Post-emit pass of static keys jump section. Convert jumps to nops. +Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() { + if (!StaticKeysJumpSection || !StaticKeysJumpSection->isFinalized()) + return Error::success(); + + const uint64_t SectionAddress = StaticKeysJumpSection->getAddress(); + DataExtractor DE(StaticKeysJumpSection->getOutputContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress); + const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table"); + uint32_t EntryID = 0; + uint64_t NumShort = 0; + uint64_t NumLong = 0; + while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) { + const uint64_t JumpAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t TargetAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t KeyAddress = + SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor); + + // Consume the status of the cursor. + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while updating static keys: %s", + toString(Cursor.takeError()).c_str()); + + ++EntryID; + + LLVM_DEBUG({ + dbgs() << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress) + << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) + << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n'; + }); + + BinaryFunction *BF = + BC.getBinaryFunctionContainingAddress(JumpAddress, + /*CheckPastEnd*/ false, + /*UseMaxSize*/ true); + assert(BF && "Cannot get function for modified static key."); + + if (!BF->isEmitted()) + continue; + + // Disassemble instruction to collect stats even if nop-conversion is + // unnecessary. + MutableArrayRef Contents = MutableArrayRef( + reinterpret_cast(BF->getImageAddress()), BF->getImageSize()); + assert(Contents.size() && "Non-empty function image expected."); + + MCInst Inst; + uint64_t Size; + const uint64_t JumpOffset = JumpAddress - BF->getAddress(); + if (!BC.DisAsm->getInstruction(Inst, Size, Contents.slice(JumpOffset), 0, + nulls())) { + llvm_unreachable("Unable to disassemble jump instruction."); + } + assert(BC.MIB->isBranch(Inst) && "Branch instruction expected."); + + if (Size == 2) + ++NumShort; + else if (Size == 5) + ++NumLong; + else + llvm_unreachable("Unexpected size for static keys jump instruction."); + + // Check if we need to convert jump instruction into a nop. + if (!NopIDs.contains(EntryID)) + continue; + + SmallString<15> NopCode; + raw_svector_ostream VecOS(NopCode); + BC.MAB->writeNopData(VecOS, Size, BC.STI.get()); + for (uint64_t I = 0; I < Size; ++I) + Contents[JumpOffset + I] = NopCode[I]; + } + + BC.outs() << "BOLT-INFO: written " << NumShort << " short and " << NumLong + << " long static keys jumps in optimized functions\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index de55fbe51764..15f95f821777 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -336,6 +336,9 @@ public: } bool isUnsupportedBranch(const MCInst &Inst) const override { + if (isDynamicBranch(Inst)) + return true; + switch (Inst.getOpcode()) { default: return false; @@ -2728,6 +2731,7 @@ public: void createUncondBranch(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { + Inst.clear(); Inst.setOpcode(X86::JMP_1); Inst.clear(); Inst.addOperand(MCOperand::createExpr( @@ -2776,6 +2780,15 @@ public: Inst.addOperand(MCOperand::createImm(CC)); } + void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC, + MCContext *Ctx) const override { + Inst.setOpcode(X86::JCC_4); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); + Inst.addOperand(MCOperand::createImm(CC)); + } + bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); diff --git a/bolt/test/X86/linux-static-keys.s b/bolt/test/X86/linux-static-keys.s new file mode 100644 index 000000000000..08454bf97631 --- /dev/null +++ b/bolt/test/X86/linux-static-keys.s @@ -0,0 +1,67 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly updates the Linux kernel static keys jump table. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr + +## Verify static keys jump bindings to instructions. + +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \ +# RUN: --bolt-info=0 |& FileCheck %s + +## Verify the bindings again on the rewritten binary with nops removed. + +# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized |& FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 static keys jump entries + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" + nop +.L0: + jmp .L1 +# CHECK: jit +# CHECK-SAME: # ID: 1 {{.*}} # Likely: 0 # InitValue: 1 + nop +.L1: + .nops 5 +# CHECK: jit +# CHECK-SAME: # ID: 2 {{.*}} # Likely: 1 # InitValue: 1 +.L2: + nop + .size _start, .-_start + + .globl foo + .type foo, %function +foo: + ret + .size foo, .-foo + + +## Static keys jump table. + .rodata + .globl __start___jump_table + .type __start___jump_table, %object +__start___jump_table: + + .long .L0 - . # Jump address + .long .L1 - . # Target address + .quad 1 # Key address + + .long .L1 - . # Jump address + .long .L2 - . # Target address + .quad 0 # Key address + + .globl __stop___jump_table + .type __stop___jump_table, %object +__stop___jump_table: + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 70a9c527b8c9857fb63a87b2d2025bf9defea7f2 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 22:28:44 +0100 Subject: [PATCH 100/933] [libc][bazel] Fix bazel build (#86190) Follow up on #86140 --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 2e8d475f196e..0d531a3dc12a 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -97,6 +97,11 @@ libc_support_library( deps = [":llvm_libc_macros_float_macros"], ) +libc_support_library( + name = "llvm_libc_macros_fcntl_macros", + hdrs = ["include/llvm-libc-macros/linux/fcntl-macros.h"], +) + ############################### Support libraries ############################## libc_support_library( @@ -3271,6 +3276,7 @@ libc_function( ":__support_common", ":__support_osutil_syscall", ":errno", + ":llvm_libc_macros_fcntl_macros", ], ) -- GitLab From 628068113710d501e88b63a1506d66dd20ce7e94 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Thu, 21 Mar 2024 14:32:13 -0700 Subject: [PATCH 101/933] [BOLT] Output basic YAML profile in BAT mode Relax assumptions that YAML output is not supported in BAT mode. Set up basic infrastructure for emitting YAML for functions not covered by BAT, such as from `.bolt.org.text` section (code identical to input binary sans external refs), or non-rewritten functions in non-relocation mode (where the function stays in the same section but BAT mapping is not emitted). This diff only produces YAML profile for non-BAT functions (skipped, non-simple). YAML profile for BAT functions is added in follow-up diffs: - https://github.com/llvm/llvm-project/pull/76911 emits YAML profile with internal control flow information only (branch profile), - https://github.com/llvm/llvm-project/pull/76896 adds cross-function profile (calls profile). Test Plan: Added bolt/test/X86/bolt-address-translation-yaml.test Reviewers: ayermolo, dcci, maksfb, rafaelauler Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76910 --- bolt/docs/BAT.md | 5 +- .../bolt/Profile/BoltAddressTranslation.h | 3 + bolt/include/bolt/Profile/DataAggregator.h | 4 + bolt/lib/Profile/DataAggregator.cpp | 68 +- bolt/lib/Rewrite/RewriteInstance.cpp | 25 +- bolt/lib/Utils/CommandLineOpts.cpp | 4 + bolt/test/X86/Inputs/blarge_new.preagg.txt | 81 + bolt/test/X86/Inputs/blarge_new.yaml | 1648 +++++++++++++++++ .../test/X86/Inputs/blarge_new_bat.preagg.txt | 79 + .../X86/bolt-address-translation-yaml.test | 40 + 10 files changed, 1936 insertions(+), 21 deletions(-) create mode 100644 bolt/test/X86/Inputs/blarge_new.preagg.txt create mode 100644 bolt/test/X86/Inputs/blarge_new.yaml create mode 100644 bolt/test/X86/Inputs/blarge_new_bat.preagg.txt create mode 100644 bolt/test/X86/bolt-address-translation-yaml.test diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 060fc632f686..186b0e5ea89d 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -14,9 +14,8 @@ binary onto the original binary. # Usage `--enable-bat` flag controls the generation of BAT section. Sampled profile needs to be passed along with the optimized binary containing BAT section to -`perf2bolt` which reads BAT section and produces fdata profile for the original -binary. Note that YAML profile generation is not supported since BAT doesn't -contain the metadata for input functions. +`perf2bolt` which reads BAT section and produces profile for the original +binary. # Internals ## Section contents diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 5f2f0959d93f..1f53f6d344ad 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,9 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// True if a given \p Address is a function with translation table entry. + bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } + private: /// Helper to update \p Map by inserting one or more BAT entries reflecting /// \p BB for function located at \p FuncAddress. At least one entry will be diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h index 5bb4d00024c5..f7840b49199f 100644 --- a/bolt/include/bolt/Profile/DataAggregator.h +++ b/bolt/include/bolt/Profile/DataAggregator.h @@ -463,6 +463,10 @@ private: /// Dump data structures into a file readable by llvm-bolt std::error_code writeAggregatedFile(StringRef OutputFilename) const; + /// Dump translated data structures into YAML + std::error_code writeBATYAML(BinaryContext &BC, + StringRef OutputFilename) const; + /// Filter out binaries based on PID void filterBinaryMMapInfo(); diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 6a64bcde911e..37c637a44a0e 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -16,6 +16,7 @@ #include "bolt/Core/BinaryFunction.h" #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Profile/Heatmap.h" +#include "bolt/Profile/YAMLProfileWriter.h" #include "bolt/Utils/CommandLineOpts.h" #include "bolt/Utils/Utils.h" #include "llvm/ADT/STLExtras.h" @@ -85,6 +86,7 @@ MaxSamples("max-samples", cl::cat(AggregatorCategory)); extern cl::opt ProfileFormat; +extern cl::opt SaveProfile; cl::opt ReadPreAggregated( "pa", cl::desc("skip perf and read data from a pre-aggregated file format"), @@ -594,10 +596,21 @@ Error DataAggregator::readProfile(BinaryContext &BC) { convertBranchData(Function); } - if (opts::AggregateOnly && - opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata) { - if (std::error_code EC = writeAggregatedFile(opts::OutputFilename)) - report_error("cannot create output data file", EC); + if (opts::AggregateOnly) { + if (opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata) + if (std::error_code EC = writeAggregatedFile(opts::OutputFilename)) + report_error("cannot create output data file", EC); + + // BAT YAML is handled by DataAggregator since normal YAML output requires + // CFG which is not available in BAT mode. + if (usesBAT()) { + if (opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) + if (std::error_code EC = writeBATYAML(BC, opts::OutputFilename)) + report_error("cannot create output data file", EC); + if (!opts::SaveProfile.empty()) + if (std::error_code EC = writeBATYAML(BC, opts::SaveProfile)) + report_error("cannot create output data file", EC); + } } return Error::success(); @@ -2258,6 +2271,53 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const { return std::error_code(); } +std::error_code DataAggregator::writeBATYAML(BinaryContext &BC, + StringRef OutputFilename) const { + std::error_code EC; + raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None); + if (EC) + return EC; + + yaml::bolt::BinaryProfile BP; + + // Fill out the header info. + BP.Header.Version = 1; + BP.Header.FileName = std::string(BC.getFilename()); + std::optional BuildID = BC.getFileBuildID(); + BP.Header.Id = BuildID ? std::string(*BuildID) : ""; + BP.Header.Origin = std::string(getReaderName()); + // Only the input binary layout order is supported. + BP.Header.IsDFSOrder = false; + // FIXME: Need to match hash function used to produce BAT hashes. + BP.Header.HashFunction = HashFunction::Default; + + ListSeparator LS(","); + raw_string_ostream EventNamesOS(BP.Header.EventNames); + for (const StringMapEntry &EventEntry : EventNames) + EventNamesOS << LS << EventEntry.first().str(); + + BP.Header.Flags = opts::BasicAggregation ? BinaryFunction::PF_SAMPLE + : BinaryFunction::PF_LBR; + + if (!opts::BasicAggregation) { + // Convert profile for functions not covered by BAT + for (auto &BFI : BC.getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; + if (!Function.hasProfile()) + continue; + if (BAT->isBATFunction(Function.getAddress())) + continue; + BP.Functions.emplace_back( + YAMLProfileWriter::convert(Function, /*UseDFS=*/false)); + } + } + + // Write the profile. + yaml::Output Out(OutFile, nullptr, 0); + Out << BP; + return std::error_code(); +} + void DataAggregator::dump() const { DataReader::dump(); } void DataAggregator::dump(const LBREntry &LBR) const { diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index cde195c17390..03f4298e817d 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -199,10 +199,7 @@ static cl::opt RelocationMode( "relocs", cl::desc("use relocations in the binary (default=autodetect)"), cl::cat(BoltCategory)); -static cl::opt -SaveProfile("w", - cl::desc("save recorded profile to a file"), - cl::cat(BoltOutputCategory)); +extern cl::opt SaveProfile; static cl::list SkipFunctionNames("skip-funcs", @@ -732,6 +729,13 @@ Error RewriteInstance::run() { // Skip disassembling if we have a translation table and we are running an // aggregation job. if (opts::AggregateOnly && BAT->enabledFor(InputFile)) { + // YAML profile in BAT mode requires CFG for .bolt.org.text functions + if (!opts::SaveProfile.empty() || + opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) { + selectFunctionsToProcess(); + disassembleFunctions(); + buildFunctionsCFG(); + } processProfileData(); return Error::success(); } @@ -2027,14 +2031,6 @@ void RewriteInstance::adjustCommandLineOptions() { if (opts::Lite) BC->outs() << "BOLT-INFO: enabling lite mode\n"; - - if (!opts::SaveProfile.empty() && BAT->enabledFor(InputFile)) { - BC->errs() - << "BOLT-ERROR: unable to save profile in YAML format for input " - "file processed by BOLT. Please remove -w option and use branch " - "profile.\n"; - exit(1); - } } namespace { @@ -3126,12 +3122,13 @@ void RewriteInstance::processProfileData() { } } - if (!opts::SaveProfile.empty()) { + if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) { YAMLProfileWriter PW(opts::SaveProfile); PW.writeProfile(*this); } if (opts::AggregateOnly && - opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) { + opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML && + !BAT->enabledFor(InputFile)) { YAMLProfileWriter PW(opts::OutputFilename); PW.writeProfile(*this); } diff --git a/bolt/lib/Utils/CommandLineOpts.cpp b/bolt/lib/Utils/CommandLineOpts.cpp index e910fa4f8672..ba296c10c00a 100644 --- a/bolt/lib/Utils/CommandLineOpts.cpp +++ b/bolt/lib/Utils/CommandLineOpts.cpp @@ -162,6 +162,10 @@ cl::opt ProfileFormat( clEnumValN(PF_YAML, "yaml", "dense YAML representation")), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltCategory)); +cl::opt SaveProfile("w", + cl::desc("save recorded profile to a file"), + cl::cat(BoltOutputCategory)); + cl::opt SplitEH("split-eh", cl::desc("split C++ exception handling code"), cl::Hidden, cl::cat(BoltOptCategory)); diff --git a/bolt/test/X86/Inputs/blarge_new.preagg.txt b/bolt/test/X86/Inputs/blarge_new.preagg.txt new file mode 100644 index 000000000000..e92f356d9188 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new.preagg.txt @@ -0,0 +1,81 @@ +B 40164b 401608 109 0 +B 401611 4017e0 115 0 +B 4017f0 401616 117 0 +B 401ba2 4015da 6 0 +B 4015d5 401b60 1 0 +B 40159a 401b60 5 0 +B 401b9d 401b70 615 2 +B 401b90 401b99 344 37 +B 401ba2 40159f 8 0 +B 4015b0 401070 9 0 +B 401544 4014a0 6 0 +B 40188a 401928 5 0 +B 40152a 4014b0 21 0 +B 40169e 40165b 2 0 +B 4014dd 401070 12 1 +B 401509 4014ec 2 2 +B 401510 401030 673 0 +B 4019de 401080 1 0 +B 401500 401070 22 0 +B 401921 4014d6 9 0 +B 4019b3 401080 3 0 +B 40162d 401070 113 0 +B 4014d1 401800 27 0 +B 401a3f 401080 1 0 +B 4018d2 401050 17 0 +B 401664 4017c0 2 0 +B 401680 401070 2 0 +B 4017d0 401669 2 0 +B 4018f7 40190d 9 0 +B 4015bc 401592 6 0 +B 401964 401090 5 0 +B 4015f8 4015cd 1 0 +B 4015ec 401070 6 0 +F 40165b 401664 2 +F 4017c0 4017d0 2 +F 401669 401680 2 +F 40190d 401921 9 +F 4014d6 4014dd 9 +F 401800 4018d2 17 +F 4018d7 4018f7 9 +F 40159f 4015b0 8 +F 401515 401544 6 +F 401070 401500 1 +F 401070 401070 157 +F 4014a0 4014d1 6 +F 401616 40162d 112 +F 4019e3 401a3f 1 +F 4014e2 401500 19 +F 401090 401090 5 +F 401030 401030 673 +F 401505 401510 668 +F 401616 4017f0 2 +F 401070 4015b0 1 +F 4015da 4015ec 6 +F 401b60 401b90 6 +F 4019b8 4019de 1 +F 401969 4019b3 3 +F 401505 401509 2 +F 401515 40152a 21 +F 401592 40159a 4 +F 401050 401050 17 +F 4015cd 4015d5 1 +F 401070 4014dd 1 +F 401b99 401ba2 8 +F 401b70 401b90 326 +F 401b99 401b9d 324 +F 401592 4015bc 1 +F 401608 401611 109 +F 401b70 401b9d 268 +F 4015b5 4015bc 5 +F 401b99 401b90 1 +F 401b70 401ba2 5 +F 401632 40164b 108 +F 401080 401080 5 +F 4014b0 4014d1 21 +F 4017e0 4017f0 115 +F 4015f1 4015f8 1 +F 401685 40169e 2 +F 401928 401964 5 +F 401800 40188a 5 +F 4014ec 401500 2 diff --git a/bolt/test/X86/Inputs/blarge_new.yaml b/bolt/test/X86/Inputs/blarge_new.yaml new file mode 100644 index 000000000000..0380f5180e90 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new.yaml @@ -0,0 +1,1648 @@ +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 + Entry: 0x4016D0 +ProgramHeaders: + - Type: PT_PHDR + Flags: [ PF_R ] + VAddr: 0x400040 + Align: 0x8 + Offset: 0x40 + - Type: PT_INTERP + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .interp + VAddr: 0x4002A8 + Offset: 0x2A8 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .rela.plt + VAddr: 0x400000 + Align: 0x1000 + Offset: 0x0 + - Type: PT_LOAD + Flags: [ PF_X, PF_R ] + FirstSec: .init + LastSec: .fini + VAddr: 0x401000 + Align: 0x1000 + Offset: 0x1000 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .rodata + LastSec: .eh_frame + VAddr: 0x402000 + Align: 0x1000 + Offset: 0x2000 + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] + FirstSec: .init_array + LastSec: .bss + VAddr: 0x403E00 + Align: 0x1000 + Offset: 0x2E00 + - Type: PT_DYNAMIC + Flags: [ PF_W, PF_R ] + FirstSec: .dynamic + LastSec: .dynamic + VAddr: 0x403E10 + Align: 0x8 + Offset: 0x2E10 + - Type: PT_NOTE + Flags: [ PF_R ] + FirstSec: .note.gnu.build-id + LastSec: .note.ABI-tag + VAddr: 0x4002C4 + Align: 0x4 + Offset: 0x2C4 + - Type: PT_GNU_EH_FRAME + Flags: [ PF_R ] + FirstSec: .eh_frame_hdr + LastSec: .eh_frame_hdr + VAddr: 0x402270 + Align: 0x4 + Offset: 0x2270 + - Type: PT_GNU_STACK + Flags: [ PF_W, PF_R ] + Align: 0x10 + Offset: 0x0 + - Type: PT_GNU_RELRO + Flags: [ PF_R ] + FirstSec: .init_array + LastSec: .got + VAddr: 0x403E00 + Offset: 0x2E00 +Sections: + - Name: .interp + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x4002A8 + AddressAlign: 0x1 + Content: 2F6C696236342F6C642D6C696E75782D7838362D36342E736F2E3200 + - Name: .note.gnu.build-id + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002C4 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: 66CF856212C3B313EA98AD840984B20EA781118A + Type: NT_PRPSINFO + - Name: .note.ABI-tag + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002E8 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: '00000000030000000200000000000000' + Type: NT_VERSION + - Name: .gnu.hash + Type: SHT_GNU_HASH + Flags: [ SHF_ALLOC ] + Address: 0x400308 + Link: .dynsym + AddressAlign: 0x8 + Header: + SymNdx: 0x1 + Shift2: 0x0 + BloomFilter: [ 0x0 ] + HashBuckets: [ 0x0 ] + HashValues: [ ] + - Name: .dynsym + Type: SHT_DYNSYM + Flags: [ SHF_ALLOC ] + Address: 0x400328 + Link: .dynstr + AddressAlign: 0x8 + - Name: .dynstr + Type: SHT_STRTAB + Flags: [ SHF_ALLOC ] + Address: 0x400430 + AddressAlign: 0x1 + - Name: .gnu.version + Type: SHT_GNU_versym + Flags: [ SHF_ALLOC ] + Address: 0x4004BA + Link: .dynsym + AddressAlign: 0x2 + Entries: [ 0, 2, 2, 3, 4, 2, 5, 5, 2, 0, 5 ] + - Name: .gnu.version_r + Type: SHT_GNU_verneed + Flags: [ SHF_ALLOC ] + Address: 0x4004D0 + Link: .dynstr + AddressAlign: 0x8 + Dependencies: + - Version: 1 + File: libm.so.6 + Entries: + - Name: GLIBC_2.2.5 + Hash: 157882997 + Flags: 0 + Other: 5 + - Name: GLIBC_2.29 + Hash: 110530953 + Flags: 0 + Other: 3 + - Version: 1 + File: libc.so.6 + Entries: + - Name: GLIBC_2.4 + Hash: 225011988 + Flags: 0 + Other: 4 + - Name: GLIBC_2.2.5 + Hash: 157882997 + Flags: 0 + Other: 2 + - Name: .rela.dyn + Type: SHT_RELA + Flags: [ SHF_ALLOC ] + Address: 0x400530 + Link: .dynsym + AddressAlign: 0x8 + Relocations: + - Offset: 0x403FF0 + Symbol: __libc_start_main + Type: R_X86_64_GLOB_DAT + - Offset: 0x403FF8 + Symbol: __gmon_start__ + Type: R_X86_64_GLOB_DAT + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Address: 0x400560 + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x404018 + Symbol: putchar + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404020 + Symbol: puts + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404028 + Symbol: pow + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404030 + Symbol: __stack_chk_fail + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404038 + Symbol: printf + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404040 + Symbol: cos + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404048 + Symbol: acos + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404050 + Symbol: sqrt + Type: R_X86_64_JUMP_SLOT + - Name: .init + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401000 + AddressAlign: 0x4 + Offset: 0x1000 + Content: F30F1EFA4883EC08488B05E92F00004885C07402FFD04883C408C3 + - Name: .plt + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401020 + AddressAlign: 0x10 + EntSize: 0x10 + Content: FF35E22F0000FF25E42F00000F1F4000FF25E22F00006800000000E9E0FFFFFFFF25DA2F00006801000000E9D0FFFFFFFF25D22F00006802000000E9C0FFFFFFFF25CA2F00006803000000E9B0FFFFFFFF25C22F00006804000000E9A0FFFFFFFF25BA2F00006805000000E990FFFFFFFF25B22F00006806000000E980FFFFFFFF25AA2F00006807000000E970FFFFFF + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x4010B0 + AddressAlign: 0x10 + Content: 4156BF082040004155415455534883EC5064488B042528000000488944244831C0E86AFFFFFF488D742430488D7C2424488B0529100000F20F101529100000F20F100D2910000066480F6ED8488B050510000066480F6EC0E8F3060000BFBF20400031C0E857FFFFFF448B5C24244585DB7E2131DBF20F1044DC30BFCA204000B8010000004883C301E832FFFFFF395C24247FE1BF0A000000E8E2FEFFFF488D742430488D7C2424488B05B10F0000F20F1015C10F0000F20F100DC10F000066480F6ED8488B058D0F000066480F6EC0E87B060000BFBF20400031C0E8DFFEFFFF448B5424244585D27E2131DBF20F1044DC30BFCA204000B8010000004883C301E8BAFEFFFF395C24247FE1BF0A000000E86AFEFFFF488B053B0F0000F20F101D630F0000488D742430F20F10155E0F0000F20F100D5E0F0000488D7C242466480F6EC0E807060000BFBF20400031C0E86BFEFFFF448B4C24244585C97E2131DBF20F1044DC30BFCA204000B8010000004883C301E846FEFFFF395C24247FE1BF0A000000E8F6FDFFFF488D742430488D7C2424488B05BD0E0000F20F101DFD0E0000F20F100DFD0E000066480F6ED066480F6EC0E896050000BFBF20400031C0E8FAFDFFFF448B4424244585C07E2131DBF20F1044DC30BFCA204000B8010000004883C301E8D5FDFFFF395C24247FE1BF0A000000E885FDFFFF488B05460E0000F20F101DA60E0000488D742430F20F100DA10E0000F20F1005A10E0000488D7C242466480F6ED0E822050000BFBF20400031C0E886FDFFFF8B7C242485FF7E2131DBF20F1044DC30BFCA204000B8010000004883C301E863FDFFFF395C24247FE1BF0A000000E813FDFFFFF20F101D530E0000F20F1015530E0000488D742430F20F100D4E0E0000F20F10054E0E0000488D7C2424E8B4040000BFBF20400031C0E818FDFFFF8B74242485F67E2131DBF20F1044DC30BFCA204000B8010000004883C301E8F5FCFFFF395C24247FE1BF0A000000E8A5FCFFFFF20F101D050E0000F20F1015050E0000488D742430F20F100D000E0000F20F1005000E0000488D7C2424E846040000BFBF20400031C0E8AAFCFFFF8B4C242485C97E2131DBF20F1044DC30BFCA204000B8010000004883C301E887FCFFFF395C24247FE1BF0A000000E837FCFFFFF20F101DB70D0000F20F1015B70D0000488D742430F20F100DB20D0000F20F1005B20D0000488D7C2424E8D8030000BFBF20400031C0E83CFCFFFF8B54242485D27E2131DBF20F1044DC30BFCA204000B8010000004883C301E819FCFFFF395C24247FE1BF0A00000041BD09000000E8C3FBFFFF488B05940C00004889442410488B05800C000041BE280000004889442418488B05660C000041BC1100000048894424080F1F00488B05490C0000BD0900000048890424F20F101C24488D742430488D7C2424F20F10542408F20F104C2418F20F10442410E82A030000BFBF20400031C0E88EFBFFFF8B44242485C07E2131DBF20F1044DC30BFCA204000B8010000004883C301E86BFBFFFF395C24247FE1BF0A000000E81BFBFFFFF20F102424F20F5C25B60C0000F20F11242483ED017584F20F102DAC0C0000F20F586C2408F20F116C24084183EC010F8556FFFFFFF20F107C2418F20F5C3D900C0000F20F117C24184183EE010F8523FFFFFFF20F103D980B0000F20F587C2410F20F117C24104183ED010F85F3FEFFFFBF3020400031DBE8AEFAFFFF4889DF488D742428E8C10500008B54242889DEBFCE20400031C04883C302E8BBFAFFFF4881FBA086010075D4BF0A000000BB6901ED3FE863FAFFFF4889DF488D742428E8860500008B5424284889DE31C0BFDF2040004883C301E87FFAFFFF4881FB6941ED3F75D3BF58204000E83CFAFFFF660FEFD2660F28C2F20F111424E8CA010000F20F101424BF80204000B802000000660F28C8660F28C2E83EFAFFFFF20F101424F20F5815B10B0000F20F103DB10B0000660F2FFA73BBBFEE204000E8E9F9FFFF660FEFD2660F28C2F20F111424E857010000F20F101424BFA0204000B802000000660F28C8660F28C2E8EBF9FFFFF20F101424F20F58156E0B0000F20F103D6E0B0000660F2FFA73BB488B442448644833042528000000750F4883C45031C05B5D415C415D415EC3E89CF9FFFF662E0F1F8400000000006690F30F1EFA31ED4989D15E4889E24883E4F0505449C7C0201C400048C7C1B01B400048C7C7B0104000FF15F2280000F490F30F1EFAC3662E0F1F84000000000090B868404000483D684040007413B8000000004885C07409BF68404000FFE06690C30F1F440000662E0F1F840000000000BE684040004881EE684040004889F048C1EE3F48C1F8034801C648D1FE7411B8000000004885C07407BF68404000FFE0C30F1F440000662E0F1F840000000000803DE1280000007517554889E5E87EFFFFFFC605CF280000015DC30F1F440000C30F1F440000662E0F1F840000000000EB8E662E0F1F8400000000000F1F4000F20F5905480A0000F20F5E05480A0000C366662E0F1F8400000000000F1F4000F20F5905300A0000F20F5E05200A0000C3662E0F1F8400000000000F1F440000F20F5EC8534889F34883EC50F20F5ED0F20F110C24DD0424660FEFC9DB3C24DB2C24F20F5ED8F20F11542418DD442418D9C1D8CAD905E6090000D8CADEE9D905E0090000DCF9F20F115C2418D9C3D8C4D8CCD8CCD9CCDEC9DECAD9CADEE1D905C4090000DC4C2418DEC1D835BC090000D9C1D8CAD8CAD9C1D8CAD8E1DD5C2418F20F10442418660F2FC80F8398000000DDD8660F2EC8660F28D0C70701000000F20F51D20F87B6010000D9C9DB7C2430F20F100D90090000DD542418F20F10442418660F540586090000DB7C2420F20F58C2E879F7FFFFF20F11442418DD442418DB6C2430D8F1DEC1DD5C2418DB6C2420D9EEDFF1DDD87714F20F107C2418660F573D59090000F20F117C2418DB2C24D8350A090000DC6C2418DD1B4883C4505BC3660F1F440000DD5C2418F20F10442418C70703000000660F28F0660F2EC8F20F51F6F20F117424180F8736010000D9C9DB7C2420DC742418DD5C2418F20F10442418E827F7FFFFDB6C2420660FEFC9F20F11442418DD5C2420F20F10542420660F2ECA660F28DAF20F51DB0F870D010000F20F102DD5070000F20F591D8D080000F20F5EC5F20F116C2430F20F115C2420E8C8F6FFFFDB2C24F20F11442440F20F10442418D83553080000F20F580563080000F20F5E442430DB3C24E89DF6FFFFF20F104C2440F20F10642420DB2C24660F28D0F20F59CCF20F59D4D9C0F20F114C2440DC6C2440DD5C2440F20F10442440F20F11542440DC6C2440DD5C2440660F164424400F1103F20F10442418F20F580507080000F20F5E442430E83CF6FFFFF20F59442420DB2C24F20F11442418DC6C2418DD5B104883C4505BC3DB7C2430F20F11542418DB7C2420E82DF6FFFFF20F10542418DB6C2430DB6C2420E926FEFFFFDB7C2430DB7C2420E80DF6FFFFDB6C2430DB6C2420E9B2FEFFFF660F28C2F20F11542448F20F115C2420E8EBF5FFFFF20F103DB3060000F20F10442418F20F105C2420F20F591D5F070000F20F5EC7F20F117C2430F20F115C2420E89AF5FFFFDB2C24F20F10742420F20F10542448D83525070000F20F59F0660F28C2F20F11742440D9C0DB3C24DC6C2440DD1BE887F5FFFFF20F10442418F20F580511070000F20F5E442430E84EF5FFFFDB2C24F20F10542448F20F59442420F20F11442440DC6C2440660F28C2DD5B08E849F5FFFFE9CFFEFFFF0F1F400041B82000000031C031D2660F1F4400004889F948C1E70248C1E91E83E103488D1491488D0C85010000004801C04839CA72074829CA4883C0014183E80175D1488906C3662E0F1F8400000000000F1F00F30F1EFA41574C8D3D4322000041564989D641554989F541544189FC55488D2D34220000534C29FD4883EC08E81FF4FFFF48C1FD03741F31DB0F1F80000000004C89F24C89EE4489E741FF14DF4883C3014839DD75EA4883C4085B5D415C415D415E415FC366662E0F1F840000000000F30F1EFAC3 + - Name: .fini + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401C28 + AddressAlign: 0x4 + Content: F30F1EFA4883EC084883C408C3 + - Name: .rodata + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x402000 + AddressAlign: 0x10 + Offset: 0x2000 + Content: 01000200000000002A2A2A2A2A2A2A2A2A2043554249432046554E4354494F4E53202A2A2A2A2A2A2A2A2A2A2A0000002A2A2A2A2A2A2A2A2A20494E54454745522053515220524F4F5453202A2A2A2A2A2A2A2A2A2A2A002A2A2A2A2A2A2A2A2A20414E474C4520434F4E56455253494F4E202A2A2A2A2A2A2A2A2A2A2A000025332E30662064656772656573203D20252E3132662072616469616E730A0000252E3132662072616469616E73203D2025332E306620646567726565730A00536F6C7574696F6E733A0020256600737172742825336429203D202532640A007371727428256C5829203D2025580A0000000000000000F0BF00000000000014400000000000002440000000000000F03F0000000000003EC0000000000000404000000000000025C0000000000000314000000000000012C00000000000003FC000000000000036400000000000000CC000000000008041C06666666666662BC00000000000002840AE47E17A14AE284000000000000008409A999999999937C00000000000001840295C8FC2F5F850C000000000000020C000000000000041400000000000001E40D7A3703D0A572140000000000080464000000000000030403333333333331540333333333333FBBF00000000000028C077BE9F1A2FDDDC3F85EB51B81E85E33F000000000000D03FFCA9F1D24D62503F0000000000807640399D52A246DF413F9B0B6097FB2119400000000000806640182D4454FB21094000004040000010410000D8410000584200000000000000C0182D4454FB211940182D4454FB212940555555555555D53FFFFFFFFFFFFFFF7F000000000000000000000000000000800000000000000000 + - Name: .eh_frame_hdr + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x402270 + AddressAlign: 0x4 + Content: 011B033B5C0000000A000000B0EDFFFFA000000040EEFFFFC800000060F4FFFF7800000090F4FFFF8C00000050F5FFFF1001000070F5FFFF2401000090F5FFFF38010000F0F8FFFF6801000040F9FFFF80010000B0F9FFFFC8010000 + - Name: .eh_frame + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x4022D0 + AddressAlign: 0x8 + Content: 1400000000000000017A5200017810011B0C070890010000100000001C000000E0F3FFFF2F000000004407101000000030000000FCF3FFFF0500000000000000240000004400000008EDFFFF90000000000E10460E184A0F0B770880003F1A3B2A33242200000000440000006C00000070EDFFFF1406000000420E108E02470E188D03420E208C04410E288605410E308306440E800103F3050A0E30430E28410E20420E18420E10420E08410B00000010000000B400000038F4FFFF110000000000000010000000C800000044F4FFFF11000000000000002C000000DC00000050F4FFFF5C03000000450E108302470E600314010A0E10410E08470B0336010A0E10410E08410B00140000000C01000080F7FFFF4300000000000000000000004400000024010000B8F7FFFF6500000000460E108F02490E188E03450E208D04450E288C05440E308606480E388307470E406E0E38410E30410E28420E20420E18420E10420E0800100000006C010000E0F7FFFF050000000000000000000000 + - Name: .init_array + Type: SHT_INIT_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E00 + AddressAlign: 0x8 + EntSize: 0x8 + Offset: 0x2E00 + Content: B017400000000000 + - Name: .fini_array + Type: SHT_FINI_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E08 + AddressAlign: 0x8 + EntSize: 0x8 + Content: '8017400000000000' + - Name: .dynamic + Type: SHT_DYNAMIC + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E10 + Link: .dynstr + AddressAlign: 0x8 + Entries: + - Tag: DT_NEEDED + Value: 0x1 + - Tag: DT_NEEDED + Value: 0x28 + - Tag: DT_INIT + Value: 0x401000 + - Tag: DT_FINI + Value: 0x401C28 + - Tag: DT_INIT_ARRAY + Value: 0x403E00 + - Tag: DT_INIT_ARRAYSZ + Value: 0x8 + - Tag: DT_FINI_ARRAY + Value: 0x403E08 + - Tag: DT_FINI_ARRAYSZ + Value: 0x8 + - Tag: DT_GNU_HASH + Value: 0x400308 + - Tag: DT_STRTAB + Value: 0x400430 + - Tag: DT_SYMTAB + Value: 0x400328 + - Tag: DT_STRSZ + Value: 0x8A + - Tag: DT_SYMENT + Value: 0x18 + - Tag: DT_DEBUG + Value: 0x0 + - Tag: DT_PLTGOT + Value: 0x404000 + - Tag: DT_PLTRELSZ + Value: 0xC0 + - Tag: DT_PLTREL + Value: 0x7 + - Tag: DT_JMPREL + Value: 0x400560 + - Tag: DT_RELA + Value: 0x400530 + - Tag: DT_RELASZ + Value: 0x30 + - Tag: DT_RELAENT + Value: 0x18 + - Tag: DT_VERNEED + Value: 0x4004D0 + - Tag: DT_VERNEEDNUM + Value: 0x2 + - Tag: DT_VERSYM + Value: 0x4004BA + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Name: .got + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403FF0 + AddressAlign: 0x8 + EntSize: 0x8 + Content: '00000000000000000000000000000000' + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404000 + AddressAlign: 0x8 + EntSize: 0x8 + Content: 103E400000000000000000000000000000000000000000003610400000000000461040000000000056104000000000006610400000000000761040000000000086104000000000009610400000000000A610400000000000 + - Name: .data + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404058 + AddressAlign: 0x8 + Content: '00000000000000000000000000000000' + - Name: .tm_clone_table + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404068 + AddressAlign: 0x8 + - Name: .bss + Type: SHT_NOBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404068 + AddressAlign: 0x1 + Size: 0x8 + - Name: .comment + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + AddressAlign: 0x1 + EntSize: 0x1 + Content: 4743433A20285562756E747520392E342E302D317562756E7475317E31362E30342920392E342E3000 + - Name: .rela.init + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init + Relocations: + - Offset: 0x40100B + Symbol: __gmon_start__ + Type: R_X86_64_REX_GOTPCRELX + Addend: -4 + - Name: .rela.text + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .text + Relocations: + - Offset: 0x4010B3 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 8 + - Offset: 0x4010D2 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4010E3 + Symbol: .LC6 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010EB + Symbol: .LC7 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010F3 + Symbol: .LC8 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010FF + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401109 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40110E + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401115 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40112C + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x40113A + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40114A + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40115B + Symbol: .LC6 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401163 + Symbol: .LC11 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40116B + Symbol: .LC12 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401177 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401181 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401186 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x40118D + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011A4 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x4011B2 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011C2 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011C9 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011D1 + Symbol: .LC13 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011DE + Symbol: .LC14 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011E6 + Symbol: .LC15 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011F5 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011FA + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401201 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401218 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401226 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401236 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401247 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40124F + Symbol: .LC16 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401257 + Symbol: .LC17 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401266 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40126B + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401272 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401289 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401297 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012A7 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012AE + Symbol: .LC2 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012B6 + Symbol: .LC18 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012C3 + Symbol: .LC19 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012CB + Symbol: .LC20 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012DA + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012DF + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4012E6 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012FB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401309 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401319 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401321 + Symbol: .LC21 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401329 + Symbol: .LC22 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401336 + Symbol: .LC23 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40133E + Symbol: .LC24 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401348 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40134D + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401354 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401369 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401377 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401387 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40138F + Symbol: .LC25 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401397 + Symbol: .LC26 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013A4 + Symbol: .LC27 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013AC + Symbol: .LC28 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013B6 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013BB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4013C2 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013D7 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x4013E5 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013F5 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013FD + Symbol: .LC29 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401405 + Symbol: .LC30 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401412 + Symbol: .LC31 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40141A + Symbol: .LC32 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401424 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401429 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401430 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401445 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401453 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401469 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401470 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40147C + Symbol: .LC3 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40148E + Symbol: .LC2 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4014A3 + Symbol: .LC0 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4014D2 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4014D7 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4014DE + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4014F3 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401501 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401511 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40151E + Symbol: .LC33 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401530 + Symbol: .LC34 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401554 + Symbol: .LC35 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40156C + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401587 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 48 + - Offset: 0x40158E + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40159B + Symbol: usqrt + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015A6 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 206 + - Offset: 0x4015B1 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015C9 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015D6 + Symbol: usqrt + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015E4 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 223 + - Offset: 0x4015ED + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015FB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 88 + - Offset: 0x401600 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401612 + Symbol: deg2rad + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40161C + Symbol: .rodata + Type: R_X86_64_32 + Addend: 128 + - Offset: 0x40162E + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40163B + Symbol: .LC41 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401643 + Symbol: .LC42 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40164E + Symbol: .rodata + Type: R_X86_64_32 + Addend: 238 + - Offset: 0x401653 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401665 + Symbol: rad2deg + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40166F + Symbol: .rodata + Type: R_X86_64_32 + Addend: 160 + - Offset: 0x401681 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40168E + Symbol: .LC45 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401696 + Symbol: .LC46 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4016C0 + Symbol: '__stack_chk_fail@@GLIBC_2.4' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4016E6 + Symbol: __libc_csu_fini + Type: R_X86_64_32S + - Offset: 0x4016ED + Symbol: __libc_csu_init + Type: R_X86_64_32S + - Offset: 0x4016F4 + Symbol: main + Type: R_X86_64_32S + - Offset: 0x4016FA + Symbol: '__libc_start_main@@GLIBC_2.2.5' + Type: R_X86_64_GOTPCRELX + Addend: -4 + - Offset: 0x401711 + Symbol: __TMC_END__ + Type: R_X86_64_32 + - Offset: 0x401717 + Symbol: .tm_clone_table + Type: R_X86_64_32S + - Offset: 0x40171E + Symbol: _ITM_deregisterTMCloneTable + Type: R_X86_64_32 + - Offset: 0x401728 + Symbol: .tm_clone_table + Type: R_X86_64_32 + - Offset: 0x401741 + Symbol: __TMC_END__ + Type: R_X86_64_32 + - Offset: 0x401748 + Symbol: .tm_clone_table + Type: R_X86_64_32S + - Offset: 0x401760 + Symbol: _ITM_registerTMCloneTable + Type: R_X86_64_32 + - Offset: 0x40176A + Symbol: .tm_clone_table + Type: R_X86_64_32 + - Offset: 0x401782 + Symbol: .bss + Type: R_X86_64_PC32 + Addend: -5 + - Offset: 0x401794 + Symbol: .bss + Type: R_X86_64_PC32 + Addend: -5 + - Offset: 0x4017C4 + Symbol: '.LC0 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017CC + Symbol: .LC1 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017E4 + Symbol: .LC1 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017EC + Symbol: '.LC0 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401836 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401840 + Symbol: '.LC1 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401860 + Symbol: '.LC2 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40186C + Symbol: '.LC3 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018B4 + Symbol: .LC9 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018C6 + Symbol: .LC10 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018D3 + Symbol: 'pow@@GLIBC_2.29' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401903 + Symbol: '.LC12 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401912 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401965 + Symbol: 'acos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401997 + Symbol: '.LC6 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40199F + Symbol: .LC5 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019B4 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4019C9 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019D1 + Symbol: '.LC7 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019DF + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A35 + Symbol: '.LC8 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401A40 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A6F + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A8F + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AB1 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AB9 + Symbol: '.LC6 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401ACD + Symbol: .LC5 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401AE2 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AF7 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401B15 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401B23 + Symbol: '.LC7 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401B2E + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401B53 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401BB9 + Symbol: __init_array_start + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401BD0 + Symbol: __init_array_end + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401BDD + Symbol: _init + Type: R_X86_64_PLT32 + Addend: -4 + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x4022F0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1568 + - Offset: 0x402304 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1616 + - Offset: 0x402340 + Symbol: .text + Type: R_X86_64_PC32 + - Offset: 0x402388 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1808 + - Offset: 0x40239C + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1840 + - Offset: 0x4023B0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1872 + - Offset: 0x4023E0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2736 + - Offset: 0x4023F8 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2816 + - Offset: 0x402440 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2928 + - Name: .rela.init_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init_array + Relocations: + - Offset: 0x403E00 + Symbol: .text + Type: R_X86_64_64 + Addend: 1792 + - Name: .rela.fini_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .fini_array + Relocations: + - Offset: 0x403E08 + Symbol: .text + Type: R_X86_64_64 + Addend: 1744 + - Type: SectionHeaderTable + Sections: + - Name: .interp + - Name: .note.gnu.build-id + - Name: .note.ABI-tag + - Name: .gnu.hash + - Name: .dynsym + - Name: .dynstr + - Name: .gnu.version + - Name: .gnu.version_r + - Name: .rela.dyn + - Name: .rela.plt + - Name: .init + - Name: .rela.init + - Name: .plt + - Name: .text + - Name: .rela.text + - Name: .fini + - Name: .rodata + - Name: .eh_frame_hdr + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .init_array + - Name: .rela.init_array + - Name: .fini_array + - Name: .rela.fini_array + - Name: .dynamic + - Name: .got + - Name: .got.plt + - Name: .data + - Name: .tm_clone_table + - Name: .bss + - Name: .comment + - Name: .symtab + - Name: .strtab + - Name: .shstrtab +Symbols: + - Name: .interp + Type: STT_SECTION + Section: .interp + Value: 0x4002A8 + - Name: .note.gnu.build-id + Type: STT_SECTION + Section: .note.gnu.build-id + Value: 0x4002C4 + - Name: .note.ABI-tag + Type: STT_SECTION + Section: .note.ABI-tag + Value: 0x4002E8 + - Name: .gnu.hash + Type: STT_SECTION + Section: .gnu.hash + Value: 0x400308 + - Name: .dynsym + Type: STT_SECTION + Section: .dynsym + Value: 0x400328 + - Name: .dynstr + Type: STT_SECTION + Section: .dynstr + Value: 0x400430 + - Name: .gnu.version + Type: STT_SECTION + Section: .gnu.version + Value: 0x4004BA + - Name: .gnu.version_r + Type: STT_SECTION + Section: .gnu.version_r + Value: 0x4004D0 + - Name: .rela.dyn + Type: STT_SECTION + Section: .rela.dyn + Value: 0x400530 + - Name: .rela.plt + Type: STT_SECTION + Section: .rela.plt + Value: 0x400560 + - Name: .init + Type: STT_SECTION + Section: .init + Value: 0x401000 + - Name: .plt + Type: STT_SECTION + Section: .plt + Value: 0x401020 + - Name: .text + Type: STT_SECTION + Section: .text + Value: 0x4010B0 + - Name: .fini + Type: STT_SECTION + Section: .fini + Value: 0x401C28 + - Name: .rodata + Type: STT_SECTION + Section: .rodata + Value: 0x402000 + - Name: .eh_frame_hdr + Type: STT_SECTION + Section: .eh_frame_hdr + Value: 0x402270 + - Name: .eh_frame + Type: STT_SECTION + Section: .eh_frame + Value: 0x4022D0 + - Name: .init_array + Type: STT_SECTION + Section: .init_array + Value: 0x403E00 + - Name: .fini_array + Type: STT_SECTION + Section: .fini_array + Value: 0x403E08 + - Name: .dynamic + Type: STT_SECTION + Section: .dynamic + Value: 0x403E10 + - Name: .got + Type: STT_SECTION + Section: .got + Value: 0x403FF0 + - Name: .got.plt + Type: STT_SECTION + Section: .got.plt + Value: 0x404000 + - Name: .data + Type: STT_SECTION + Section: .data + Value: 0x404058 + - Name: .tm_clone_table + Type: STT_SECTION + Section: .tm_clone_table + Value: 0x404068 + - Name: .bss + Type: STT_SECTION + Section: .bss + Value: 0x404068 + - Name: .comment + Type: STT_SECTION + Section: .comment + - Name: basicmath_large.c + Type: STT_FILE + Index: SHN_ABS + - Name: .LC6 + Section: .rodata + Value: 0x402110 + - Name: .LC7 + Section: .rodata + Value: 0x402118 + - Name: .LC8 + Section: .rodata + Value: 0x402120 + - Name: .LC4 + Section: .rodata + Value: 0x402108 + - Name: .LC11 + Section: .rodata + Value: 0x402128 + - Name: .LC12 + Section: .rodata + Value: 0x402130 + - Name: .LC13 + Section: .rodata + Value: 0x402138 + - Name: .LC14 + Section: .rodata + Value: 0x402140 + - Name: .LC15 + Section: .rodata + Value: 0x402148 + - Name: .LC16 + Section: .rodata + Value: 0x402150 + - Name: .LC17 + Section: .rodata + Value: 0x402158 + - Name: .LC2 + Section: .rodata + Value: 0x4020F8 + - Name: .LC18 + Section: .rodata + Value: 0x402160 + - Name: .LC19 + Section: .rodata + Value: 0x402168 + - Name: .LC20 + Section: .rodata + Value: 0x402170 + - Name: .LC21 + Section: .rodata + Value: 0x402178 + - Name: .LC22 + Section: .rodata + Value: 0x402180 + - Name: .LC23 + Section: .rodata + Value: 0x402188 + - Name: .LC24 + Section: .rodata + Value: 0x402190 + - Name: .LC25 + Section: .rodata + Value: 0x402198 + - Name: .LC26 + Section: .rodata + Value: 0x4021A0 + - Name: .LC27 + Section: .rodata + Value: 0x4021A8 + - Name: .LC28 + Section: .rodata + Value: 0x4021B0 + - Name: .LC29 + Section: .rodata + Value: 0x4021B8 + - Name: .LC30 + Section: .rodata + Value: 0x4021C0 + - Name: .LC31 + Section: .rodata + Value: 0x4021C8 + - Name: .LC32 + Section: .rodata + Value: 0x4021D0 + - Name: .LC3 + Section: .rodata + Value: 0x402100 + - Name: .LC0 + Section: .rodata + Value: 0x4020F0 + - Name: .LC33 + Section: .rodata + Value: 0x4021D8 + - Name: .LC34 + Section: .rodata + Value: 0x4021E0 + - Name: .LC35 + Section: .rodata + Value: 0x4021E8 + - Name: .LC41 + Section: .rodata + Value: 0x4021F0 + - Name: .LC42 + Section: .rodata + Value: 0x4021F8 + - Name: .LC45 + Section: .rodata + Value: 0x402200 + - Name: .LC46 + Section: .rodata + Value: 0x402208 + - Name: crtstuff.c + Type: STT_FILE + Index: SHN_ABS + - Name: __TMC_LIST__ + Type: STT_OBJECT + Section: .tm_clone_table + Value: 0x404068 + - Name: deregister_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x401710 + - Name: register_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x401740 + - Name: __do_global_dtors_aux + Type: STT_FUNC + Section: .text + Value: 0x401780 + - Name: completed.8023 + Type: STT_OBJECT + Section: .bss + Value: 0x404068 + Size: 0x1 + - Name: __do_global_dtors_aux_fini_array_entry + Type: STT_OBJECT + Section: .fini_array + Value: 0x403E08 + - Name: frame_dummy + Type: STT_FUNC + Section: .text + Value: 0x4017B0 + - Name: __frame_dummy_init_array_entry + Type: STT_OBJECT + Section: .init_array + Value: 0x403E00 + - Name: rad2deg.c + Type: STT_FILE + Index: SHN_ABS + - Name: '.LC0 (1)' + Section: .rodata + Value: 0x402210 + - Name: .LC1 + Section: .rodata + Value: 0x402218 + - Name: cubic.c + Type: STT_FILE + Index: SHN_ABS + - Name: '.LC0 (2)' + Section: .rodata + Value: 0x402220 + - Name: '.LC1 (1)' + Section: .rodata + Value: 0x402224 + - Name: '.LC2 (1)' + Section: .rodata + Value: 0x402228 + - Name: '.LC3 (1)' + Section: .rodata + Value: 0x40222C + - Name: .LC9 + Section: .rodata + Value: 0x402248 + - Name: .LC10 + Section: .rodata + Value: 0x402250 + - Name: '.LC12 (1)' + Section: .rodata + Value: 0x402260 + - Name: '.LC6 (1)' + Section: .rodata + Value: 0x402170 + - Name: .LC5 + Section: .rodata + Value: 0x402230 + - Name: '.LC7 (1)' + Section: .rodata + Value: 0x402238 + - Name: '.LC8 (1)' + Section: .rodata + Value: 0x402240 + - Name: isqrt.c + Type: STT_FILE + Index: SHN_ABS + - Name: 'crtstuff.c (1)' + Type: STT_FILE + Index: SHN_ABS + - Name: __FRAME_END__ + Type: STT_OBJECT + Section: .eh_frame + Value: 0x40244C + - Type: STT_FILE + Index: SHN_ABS + - Name: __init_array_end + Section: .init_array + Value: 0x403E08 + - Name: _DYNAMIC + Type: STT_OBJECT + Section: .dynamic + Value: 0x403E10 + - Name: __init_array_start + Section: .init_array + Value: 0x403E00 + - Name: __GNU_EH_FRAME_HDR + Section: .eh_frame_hdr + Value: 0x402270 + - Name: _GLOBAL_OFFSET_TABLE_ + Type: STT_OBJECT + Section: .got.plt + Value: 0x404000 + - Name: __libc_csu_fini + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401C20 + Size: 0x5 + - Name: 'putchar@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _ITM_deregisterTMCloneTable + Binding: STB_WEAK + - Name: data_start + Section: .data + Binding: STB_WEAK + Value: 0x404058 + - Name: 'puts@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: usqrt + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401B60 + Size: 0x43 + - Name: _edata + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x404068 + - Name: 'pow@@GLIBC_2.29' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _fini + Type: STT_FUNC + Section: .fini + Binding: STB_GLOBAL + Value: 0x401C28 + Other: [ STV_HIDDEN ] + - Name: '__stack_chk_fail@@GLIBC_2.4' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'printf@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'cos@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'acos@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: '__libc_start_main@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: deg2rad + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4017E0 + Size: 0x11 + - Name: __data_start + Section: .data + Binding: STB_GLOBAL + Value: 0x404058 + - Name: SolveCubic + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401800 + Size: 0x35C + - Name: __gmon_start__ + Binding: STB_WEAK + - Name: __dso_handle + Type: STT_OBJECT + Section: .data + Binding: STB_GLOBAL + Value: 0x404060 + Other: [ STV_HIDDEN ] + - Name: _IO_stdin_used + Type: STT_OBJECT + Section: .rodata + Binding: STB_GLOBAL + Value: 0x402000 + Size: 0x4 + - Name: __libc_csu_init + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401BB0 + Size: 0x65 + - Name: _end + Section: .bss + Binding: STB_GLOBAL + Value: 0x404070 + - Name: _dl_relocate_static_pie + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401700 + Size: 0x5 + Other: [ STV_HIDDEN ] + - Name: _start + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4016D0 + Size: 0x2F + - Name: rad2deg + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4017C0 + Size: 0x11 + - Name: __bss_start + Section: .bss + Binding: STB_GLOBAL + Value: 0x404068 + - Name: main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4010B0 + Size: 0x614 + - Name: __TMC_END__ + Type: STT_OBJECT + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x404068 + Other: [ STV_HIDDEN ] + - Name: _ITM_registerTMCloneTable + Binding: STB_WEAK + - Name: 'sqrt@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _init + Type: STT_FUNC + Section: .init + Binding: STB_GLOBAL + Value: 0x401000 + Other: [ STV_HIDDEN ] +DynamicSymbols: + - Name: putchar + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: puts + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pow + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __stack_chk_fail + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: printf + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: cos + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: acos + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __libc_start_main + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __gmon_start__ + Binding: STB_WEAK + - Name: sqrt + Type: STT_FUNC + Binding: STB_GLOBAL +... diff --git a/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt b/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt new file mode 100644 index 000000000000..e9e4553aad95 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt @@ -0,0 +1,79 @@ +B 40169e 40165b 7 0 +B 401664 800012 7 0 +B 401680 401070 7 0 +B 800022 401669 7 0 +B 401611 800000 121 0 +B 40162d 401070 119 0 +B 4015d5 800040 2 0 +B 800080 4015da 6 0 +B 40164b 401608 115 0 +B 800080 40159f 24 0 +B 4015ec 401070 6 0 +B 8001d0 401090 1 0 +B 4014d1 800082 25 0 +B 401510 401030 616 0 +B 8002ab 401080 1 0 +B 80007b 80004c 483 1 +B 800072 80004c 597 77 +B 80010c 800194 1 0 +B 401509 4014ec 1 0 +B 800010 401616 119 0 +B 80024a 401080 1 0 +B 800154 401050 20 0 +B 4014dd 401070 9 0 +B 80021f 401080 1 0 +B 800193 4014d6 8 0 +B 40159a 800040 19 0 +B 4015f8 4015cd 2 0 +B 40152a 4014b0 24 0 +B 401500 401070 15 0 +B 4015bc 401592 21 0 +B 401544 4014a0 1 0 +B 80004a 800052 24 0 +B 4015b0 401070 20 0 +B 800050 80007d 29 29 +F 401685 40169e 7 +F 4014a0 4014d1 1 +F 401090 401090 1 +F 401050 401050 20 +F 40159f 4015b0 20 +F 80007d 800080 27 +F 401515 401544 1 +F 4014e2 401500 13 +F 401592 40159a 19 +F 401505 401509 1 +F 4014b0 4014d1 24 +F 800194 8001d0 1 +F 8001d5 80021f 1 +F 401616 40162d 114 +F 80024f 8002ab 1 +F 800159 800193 7 +F 80004c 800050 26 +F 800224 80024a 1 +F 800082 80010c 1 +F 401080 401080 3 +F 401070 401070 168 +F 80004c 800072 555 +F 401616 800010 2 +F 401070 40162d 4 +F 800082 800154 20 +F 401669 401680 7 +F 40159f 800080 1 +F 4014ec 401500 1 +F 800012 800022 7 +F 401030 401030 616 +F 80004c 80007b 473 +F 800052 800072 24 +F 800040 80004a 21 +F 4015b5 4015bc 18 +F 4015cd 4015d5 2 +F 401592 4015bc 1 +F 4015da 4015ec 6 +F 4015f1 4015f8 2 +F 800000 800010 116 +F 401608 401611 115 +F 401632 40164b 114 +F 401515 40152a 24 +F 40165b 401664 7 +F 401505 401510 612 +F 4014d6 4014dd 8 diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test new file mode 100644 index 000000000000..25ff4e7fbfcc --- /dev/null +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -0,0 +1,40 @@ +# Check new BAT format containing hashes for YAML profile. + +RUN: yaml2obj %p/Inputs/blarge_new.yaml &> %t.exe +RUN: llvm-bolt %t.exe -o %t.out --pa -p %p/Inputs/blarge_new.preagg.txt \ +RUN: --reorder-blocks=ext-tsp --split-functions --split-strategy=cdsplit \ +RUN: --reorder-functions=cdsort --enable-bat --dyno-stats --skip-funcs=main \ +RUN: 2>&1 | FileCheck --check-prefix WRITE-BAT-CHECK %s +RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat.preagg.txt -w %t.yaml -o %t.fdata \ +RUN: 2>&1 | FileCheck --check-prefix READ-BAT-CHECK %s +RUN: FileCheck --input-file %t.yaml --check-prefix YAML-BAT-CHECK %s +# Check that YAML converted from fdata matches YAML created directly with BAT. +RUN: llvm-bolt %t.exe -data %t.fdata -w %t.yaml-fdata -o /dev/null +RUN: FileCheck --input-file %t.yaml-fdata --check-prefix YAML-BAT-CHECK %s + +# Test resulting YAML profile with the original binary (no-stale mode) +RUN: llvm-bolt %t.exe -data %t.yaml -o %t.null -dyno-stats \ +RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s + +WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps +WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 + +READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT +READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries +READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries + +YAML-BAT-CHECK: functions: +YAML-BAT-CHECK: - name: main +YAML-BAT-CHECK-NEXT: fid: 2 +YAML-BAT-CHECK-NEXT: hash: 0x9895746D48B2C876 +YAML-BAT-CHECK-NEXT: exec: 0 +YAML-BAT-CHECK-NEXT: nblocks: 46 +YAML-BAT-CHECK-NEXT: blocks: +YAML-BAT-CHECK-NEXT: - bid: 0 +YAML-BAT-CHECK-NEXT: insns: 26 +YAML-BAT-CHECK-NEXT: hash: 0xA900AE79CFD40000 +YAML-BAT-CHECK-NEXT: succ: [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ] + +CHECK-BOLT-YAML: pre-processing profile using YAML profile reader +CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile -- GitLab From cde54df39cab3a1d60a3e1862ab341609bee3cc3 Mon Sep 17 00:00:00 2001 From: Cooper Partin Date: Thu, 21 Mar 2024 14:43:15 -0700 Subject: [PATCH 102/933] Add support for PSV EntryFunctionName (#84409) This change introduces a version 3 of the PSV data that includes support for the name of the entry function as an offset into StringTable data to a null-terminated utf-8 string. Additional tests were added to ensure that the new value was properly serialized/deserialized from object data. Fixes #80175 --------- Co-authored-by: Cooper Partin --- llvm/include/llvm/BinaryFormat/DXContainer.h | 13 +++ llvm/include/llvm/MC/DXContainerPSVInfo.h | 25 ++-- llvm/include/llvm/MC/StringTableBuilder.h | 8 +- llvm/include/llvm/Object/DXContainer.h | 16 ++- .../include/llvm/ObjectYAML/DXContainerYAML.h | 5 +- llvm/lib/MC/DXContainerPSVInfo.cpp | 75 ++++++++---- llvm/lib/Object/DXContainer.cpp | 15 ++- llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 3 +- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 15 +++ .../DXContainer/PSVv3-amplification.yaml | 97 ++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-compute.yaml | 95 +++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-domain.yaml | 105 +++++++++++++++++ .../DXContainer/PSVv3-geometry.yaml | 105 +++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-hull.yaml | 107 +++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-mesh.yaml | 109 ++++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-pixel.yaml | 99 ++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-vertex.yaml | 97 ++++++++++++++++ llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 3 + 18 files changed, 940 insertions(+), 52 deletions(-) create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index 532f9481766a..ba882c4a6f32 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -424,6 +424,19 @@ struct ResourceBindInfo : public v0::ResourceBindInfo { }; } // namespace v2 + +namespace v3 { +struct RuntimeInfo : public v2::RuntimeInfo { + uint32_t EntryNameOffset; + + void swapBytes() { sys::swapByteOrder(EntryNameOffset); } + + void swapBytes(Triple::EnvironmentType Stage) { + v2::RuntimeInfo::swapBytes(Stage); + } +}; + +} // namespace v3 } // namespace PSV #define COMPONENT_PRECISION(Val, Enum) Enum = Val, diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h index 7d21c18d252f..bad2fe78eb8f 100644 --- a/llvm/include/llvm/MC/DXContainerPSVInfo.h +++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h @@ -9,9 +9,11 @@ #ifndef LLVM_MC_DXCONTAINERPSVINFO_H #define LLVM_MC_DXCONTAINERPSVINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" +#include "llvm/MC/StringTableBuilder.h" #include "llvm/TargetParser/Triple.h" #include @@ -45,8 +47,9 @@ struct PSVSignatureElement { // modifiable format, and can be used to serialize the data back into valid PSV // RuntimeInfo. struct PSVRuntimeInfo { + PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {} bool IsFinalized = false; - dxbc::PSV::v2::RuntimeInfo BaseData; + dxbc::PSV::v3::RuntimeInfo BaseData; SmallVector Resources; SmallVector InputElements; SmallVector OutputElements; @@ -64,6 +67,7 @@ struct PSVRuntimeInfo { std::array, 4> InputOutputMap; SmallVector InputPatchMap; SmallVector PatchOutputMap; + llvm::StringRef EntryName; // Serialize PSVInfo into the provided raw_ostream. The version field // specifies the data version to encode, the default value specifies encoding @@ -71,19 +75,12 @@ struct PSVRuntimeInfo { void write(raw_ostream &OS, uint32_t Version = std::numeric_limits::max()) const; - void finalize(Triple::EnvironmentType Stage) { - IsFinalized = true; - BaseData.SigInputElements = static_cast(InputElements.size()); - BaseData.SigOutputElements = static_cast(OutputElements.size()); - BaseData.SigPatchOrPrimElements = - static_cast(PatchOrPrimElements.size()); - if (!sys::IsBigEndianHost) - return; - BaseData.swapBytes(); - BaseData.swapBytes(Stage); - for (auto &Res : Resources) - Res.swapBytes(); - } + void finalize(Triple::EnvironmentType Stage); + +private: + SmallVector IndexBuffer; + SmallVector SignatureElements; + StringTableBuilder DXConStrTabBuilder; }; class Signature { diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index 4ee421e22c17..a738683548cf 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -74,12 +74,8 @@ public: /// Check if a string is contained in the string table. Since this class /// doesn't store the string values, this function can be used to check if /// storage needs to be done prior to adding the string. - bool contains(StringRef S) const { - return contains(CachedHashStringRef(S)); - } - bool contains(CachedHashStringRef S) const { - return StringIndexMap.count(S); - } + bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); } + bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); } size_t getSize() const { return Size; } void clear(); diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index b6e3d321da24..19c83ba6c6e8 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -125,7 +125,8 @@ class PSVRuntimeInfo { uint32_t Size; using InfoStruct = std::variant; + dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo, + dxbc::PSV::v3::RuntimeInfo>; InfoStruct BasicInfo; ResourceArray Resources; StringRef StringTable; @@ -151,9 +152,11 @@ public: ResourceArray getResources() const { return Resources; } uint32_t getVersion() const { - return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) - ? 2 - : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0); + return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo) + ? 3 + : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) ? 2 + : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1 + : 0); } uint32_t getResourceStride() const { return Resources.Stride; } @@ -161,6 +164,11 @@ public: const InfoStruct &getInfo() const { return BasicInfo; } template const T *getInfoAs() const { + if (const auto *P = std::get_if(&BasicInfo)) + return static_cast(P); + if (std::is_same::value) + return nullptr; + if (const auto *P = std::get_if(&BasicInfo)) return static_cast(P); if (std::is_same::value) diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index f7f8d5e6bf47..9c4d9e19f11b 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -107,7 +107,7 @@ struct PSVInfo { // the format. uint32_t Version; - dxbc::PSV::v2::RuntimeInfo Info; + dxbc::PSV::v3::RuntimeInfo Info; uint32_t ResourceStride; SmallVector Resources; SmallVector SigInputElements; @@ -121,12 +121,15 @@ struct PSVInfo { MaskVector InputPatchMap; MaskVector PatchOutputMap; + StringRef EntryName; + void mapInfoForVersion(yaml::IO &IO); PSVInfo(); PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage); PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P); PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P); + PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable); }; struct SignatureParameter { diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp index 48182fcd31df..aeff69380139 100644 --- a/llvm/lib/MC/DXContainerPSVInfo.cpp +++ b/llvm/lib/MC/DXContainerPSVInfo.cpp @@ -81,13 +81,18 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo); break; case 2: - default: InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); + break; + case 3: + default: + InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo); + BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); } - // Write the size of the info. + // Write the size of the info. support::endian::write(OS, InfoSize, llvm::endianness::little); + // Write the info itself. OS.write(reinterpret_cast(&BaseData), InfoSize); @@ -104,32 +109,12 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { if (Version == 0) return; - StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer)); - SmallVector IndexBuffer; - SmallVector SignatureElements; - SmallVector SemanticNames; - - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, InputElements); - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, OutputElements); - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, PatchOrPrimElements); - - StrTabBuilder.finalize(); - for (auto ElAndName : zip(SignatureElements, SemanticNames)) { - v0::SignatureElement &El = std::get<0>(ElAndName); - StringRef Name = std::get<1>(ElAndName); - El.NameOffset = static_cast(StrTabBuilder.getOffset(Name)); - if (sys::IsBigEndianHost) - El.swapBytes(); - } - - support::endian::write(OS, static_cast(StrTabBuilder.getSize()), + support::endian::write(OS, + static_cast(DXConStrTabBuilder.getSize()), llvm::endianness::little); // Write the string table. - StrTabBuilder.write(OS); + DXConStrTabBuilder.write(OS); // Write the index table size, then table. support::endian::write(OS, static_cast(IndexBuffer.size()), @@ -162,6 +147,46 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { llvm::endianness::little); } +void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) { + IsFinalized = true; + BaseData.SigInputElements = static_cast(InputElements.size()); + BaseData.SigOutputElements = static_cast(OutputElements.size()); + BaseData.SigPatchOrPrimElements = + static_cast(PatchOrPrimElements.size()); + + SmallVector SemanticNames; + + // Build a string table and set associated offsets to be written when + // write() is called + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, InputElements); + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, OutputElements); + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, PatchOrPrimElements); + + DXConStrTabBuilder.add(EntryName); + + DXConStrTabBuilder.finalize(); + for (auto ElAndName : zip(SignatureElements, SemanticNames)) { + llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName); + StringRef Name = std::get<1>(ElAndName); + El.NameOffset = static_cast(DXConStrTabBuilder.getOffset(Name)); + if (sys::IsBigEndianHost) + El.swapBytes(); + } + + BaseData.EntryNameOffset = + static_cast(DXConStrTabBuilder.getOffset(EntryName)); + + if (!sys::IsBigEndianHost) + return; + BaseData.swapBytes(); + BaseData.swapBytes(Stage); + for (auto &Res : Resources) + Res.swapBytes(); +} + void Signature::write(raw_ostream &OS) { SmallVector SigParams; SigParams.reserve(Params.size()); diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 935749afe338..3b1a6203a1f8 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -247,7 +247,14 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { const uint32_t PSVVersion = getVersion(); // Detect the PSVVersion by looking at the size field. - if (PSVVersion == 2) { + if (PSVVersion == 3) { + v3::RuntimeInfo Info; + if (Error Err = readStruct(PSVInfoData, Current, Info)) + return Err; + if (sys::IsBigEndianHost) + Info.swapBytes(ShaderStage); + BasicInfo = Info; + } else if (PSVVersion == 2) { v2::RuntimeInfo Info; if (Error Err = readStruct(PSVInfoData, Current, Info)) return Err; @@ -425,6 +432,8 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { } uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -433,6 +442,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -441,6 +452,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index 09a5e41c7123..f3a518df3175 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -198,8 +198,9 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { if (!P.Info.has_value()) continue; mcdxbc::PSVRuntimeInfo PSV; - memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo)); + memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo)); PSV.Resources = P.Info->Resources; + PSV.EntryName = P.Info->EntryName; for (auto El : P.Info->SigInputElements) PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{ diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index a6871e7855e4..38063670aee6 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -74,6 +74,16 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P) memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo)); } +DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, + StringRef StringTable) + : Version(3), + EntryName(StringTable.substr(P->EntryNameOffset, + StringTable.find('\0', P->EntryNameOffset) - + P->EntryNameOffset)) { + memset(&Info, 0, sizeof(Info)); + memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo)); +} + namespace yaml { void MappingTraits::mapping( @@ -348,6 +358,11 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) { IO.mapRequired("NumThreadsX", Info.NumThreadsX); IO.mapRequired("NumThreadsY", Info.NumThreadsY); IO.mapRequired("NumThreadsZ", Info.NumThreadsZ); + + if (Version == 2) + return; + + IO.mapRequired("EntryName", EntryName); } } // namespace llvm diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml new file mode 100644 index 000000000000..09885bd529f0 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml @@ -0,0 +1,97 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 14 + PayloadSizeInBytes: 4092 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: ASEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 14 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 14 +# CHECK-NEXT: PayloadSizeInBytes: 4092 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: ASEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml new file mode 100644 index 000000000000..ee6fb112c772 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml @@ -0,0 +1,95 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 5 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: CSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 5 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 5 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: CSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml new file mode 100644 index 000000000000..dd367deae88e --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml @@ -0,0 +1,105 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 4 + InputControlPointCount: 1024 + OutputPositionPresent: 1 + TessellatorDomain: 2056 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPatchConstOrPrimVectors: 0 + SigInputVectors: 0 + SigOutputVectors: [ 0, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: DSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + PatchOutputMap: [] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 4 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 4 +# CHECK-NEXT: InputControlPointCount: 1024 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: TessellatorDomain: 2056 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: DSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: PatchOutputMap: [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml new file mode 100644 index 000000000000..4c7680b63b02 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml @@ -0,0 +1,105 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 2 + InputPrimitive: 1024 + OutputTopology: 4096 + OutputStreamMask: 2056 + OutputPositionPresent: 1 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + MaxVertexCount: 4096 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: GSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 2 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 2 +# CHECK-NEXT: InputPrimitive: 1024 +# CHECK-NEXT: OutputTopology: 4096 +# CHECK-NEXT: OutputStreamMask: 2056 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: MaxVertexCount: 4096 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: GSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml new file mode 100644 index 000000000000..3bbad8a9b0ee --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml @@ -0,0 +1,107 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 3 + InputControlPointCount: 1024 + OutputControlPointCount: 4096 + TessellatorDomain: 2056 + TessellatorOutputPrimitive: 8192 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPatchConstOrPrimVectors: 0 + SigInputVectors: 0 + SigOutputVectors: [ 0, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: HSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + InputPatchMap: [] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 3 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 3 +# CHECK-NEXT: InputControlPointCount: 1024 +# CHECK-NEXT: OutputControlPointCount: 4096 +# CHECK-NEXT: TessellatorDomain: 2056 +# CHECK-NEXT: TessellatorOutputPrimitive: 8192 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: HSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: InputPatchMap: [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml new file mode 100644 index 000000000000..c5ea1fcf0780 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml @@ -0,0 +1,109 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 13 + GroupSharedBytesUsed: 1024 + GroupSharedBytesDependentOnViewID: 2056 + PayloadSizeInBytes: 4092 + MaxOutputVertices: 8196 + MaxOutputPrimitives: 4092 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPrimVectors: 128 + MeshOutputTopology: 16 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: MSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 13 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 13 +# CHECK-NEXT: GroupSharedBytesUsed: 1024 +# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056 +# CHECK-NEXT: PayloadSizeInBytes: 4092 +# CHECK-NEXT: MaxOutputVertices: 8196 +# CHECK-NEXT: MaxOutputPrimitives: 4092 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPrimVectors: 128 +# CHECK-NEXT: MeshOutputTopology: 16 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: MSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml new file mode 100644 index 000000000000..b28d5ec8074d --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml @@ -0,0 +1,99 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 0 + DepthOutput: 7 + SampleFrequency: 96 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: PSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 0 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 0 +# CHECK-NEXT: DepthOutput: 7 +# CHECK-NEXT: SampleFrequency: 96 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: PSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml new file mode 100644 index 000000000000..d1fb55839931 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml @@ -0,0 +1,97 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 1 + OutputPositionPresent: 1 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: VSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 1 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 1 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: VSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index 69d9b9a2f784..ec4f5c74498f 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -99,6 +99,9 @@ dumpDXContainer(MemoryBufferRef Source) { else if (const auto *P = std::get_if(&PSVInfo->getInfo())) NewPart.Info = DXContainerYAML::PSVInfo(P); + else if (const auto *P = + std::get_if(&PSVInfo->getInfo())) + NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable()); NewPart.Info->ResourceStride = PSVInfo->getResourceStride(); for (auto Res : PSVInfo->getResources()) NewPart.Info->Resources.push_back(Res); -- GitLab From b19bf3e888f95c35bf641cd0eee18a8da702f6fe Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Thu, 21 Mar 2024 15:48:35 -0600 Subject: [PATCH 103/933] [clang][SPIRV] Don't warn on -mcmodel (#86039) The code model doesn't affect the sub-compilation, so don't check it. Followup to #70740. --- clang/lib/Driver/ToolChains/Clang.cpp | 4 ++-- clang/test/Driver/unsupported-option-gpu.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bc9cc8ce6cf5..86a287db72a4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { - // NVPTX/AMDGPU does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { + // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index f23cb71ebfb0..5618b2cba72e 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,4 +2,5 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all +// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc -- GitLab From b609a4d7ea8b716f5f0ec83d10945362f42e730d Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:50:44 -0700 Subject: [PATCH 104/933] [lld-macho][NFC] Refactor insertions into inputSections (#85692) Before this change, after `InputSection` objects are created, they need to be added to the appropriate container for tracking. The logic for selecting the appropriate container lives in `Driver.cpp` / `gatherInputSections`, where the `InputSection` is added to the matching container depending on the input config and the type of `InputSection`. Also, multiple other locations also insert directly into `inputSections` array - assuming that that is the appropriate container for the `InputSection`'s they create. Currently this is the correct assumption, however an upcoming feature will change this. For an upcoming feature (relative method lists), we need to route `InputSection`'s either to `inputSections` array or to a synthetic section, depending on weather the relative method list optimization is enabled or not. We can achieve the above either by duplicating some of the logic or refactoring the routing and `InputSection`'s and reusing that. The refactoring & code sharing approach seems the correct way to go - as such this diff performs the refactoring while not introducing any functional changes. Later on we can just call `addInputSection` and not have to worry about routing logic. --------- --- lld/MachO/Driver.cpp | 42 ++++----------------------------- lld/MachO/InputSection.cpp | 38 +++++++++++++++++++++++++++++ lld/MachO/InputSection.h | 3 +++ lld/MachO/ObjC.cpp | 10 ++++---- lld/MachO/SyntheticSections.cpp | 4 ++-- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 36248925d65a..919a14b8bcf0 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - inputSections.push_back(isec); + addInputSection(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,53 +1220,18 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); - int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - ConcatOutputSection *osec = nullptr; - for (const Subsection &subsection : section->subsections) { - if (auto *isec = dyn_cast(subsection.isec)) { - if (isec->isCoalescedWeak()) - continue; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - continue; - } - isec->outSecOff = inputOrder++; - if (!osec) - osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - } + for (const Subsection &subsection : section->subsections) + addInputSection(subsection.isec); } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } - assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1422,6 +1387,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); + inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 8f5affb1dc21..22930d52dd1d 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,6 +37,44 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; +int macho::inputSectionsOrder = 0; + +// Call this function to add a new InputSection and have it routed to the +// appropriate container. Depending on its type and current config, it will +// either be added to 'inputSections' vector or to a synthetic section. +void lld::macho::addInputSection(InputSection *inputSection) { + if (auto *isec = dyn_cast(inputSection)) { + if (isec->isCoalescedWeak()) + return; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + return; + } + isec->outSecOff = inputSectionsOrder++; + auto *osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(inputSection)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputSectionsOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSectionsOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = dyn_cast(inputSection)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSectionsOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + + assert(inputSectionsOrder <= UnspecifiedInputOrder); +} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index b25f0638f4c6..694bdf734907 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,6 +302,8 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; +// This is used as a counter for specyfing input order for input sections +extern int inputSectionsOrder; namespace section_names { @@ -369,6 +371,7 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names +void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 40df2243b26f..66959cfb665f 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - allInputSections.push_back(newCatList); + addInputSection(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - allInputSections.push_back(newBodySec); + addInputSection(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 7ee3261ce307..1b3694528de1 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - inputSections.push_back(in.imageLoaderCache); + addInputSection(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - inputSections.push_back(objcSelref); + addInputSection(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From e4a672ef85f76c3402b81640e1e83e5d3069d1b9 Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:53:09 -0700 Subject: [PATCH 105/933] [lld][macho] Fix gcc category merging warning (#86091) Fixing gcc warning regarding creating non-null-terminated string: ``` ../../lld/MachO/ObjC.cpp:1226:10: warning: 'char* strncpy(char*, const char*, size_t)' output truncated before terminating nul copying as many bytes from a string as its length [-Wstringop-truncation] 1226 | strncpy(strData, str, len); | ~~~~~~~^~~~~~~~~~~~~~~~~~~ ../../lld/MachO/ObjC.cpp: In member function 'void {anonymous}::ObjcCategoryMerger::emitAndLinkPointerList(lld::macho::Defined*, uint32_t, const {anonymous}::ObjcCategoryMerger::ClassExtensionInfo&, const {anonymous}::ObjcCategoryMerger::PointerListInfo&)': ../../lld/MachO/ObjC.cpp:1223:24: note: length computed here 1223 | uint32_t len = strlen(str); | ~~~~~~^~~~~ ``` This is not actually a bug, as `newSectionData` returns a zero-initialized memory region, so the null terminator will be there. --- lld/MachO/ObjC.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 66959cfb665f..5902b82d30f5 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -1221,9 +1221,11 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - auto &data = newSectionData(len + 1); + uint32_t bufSize = len + 1; + auto &data = newSectionData(bufSize); char *strData = reinterpret_cast(data.data()); - strncpy(strData, str, len); + // Copy the string chars and null-terminator + memcpy(strData, str, bufSize); return StringRef(strData, len); } -- GitLab From e470ca89ba77b2f200ff3a8ad65c74028f42c5f7 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:03:34 -0700 Subject: [PATCH 106/933] [InstallAPI] Report exports discovered in binary but not in interface (#86025) This patch completes the classes of errors installapi can detect. --- .../clang/Basic/DiagnosticInstallAPIKinds.td | 1 + .../include/clang/InstallAPI/DylibVerifier.h | 13 +- clang/lib/InstallAPI/DylibVerifier.cpp | 171 ++++++- clang/test/InstallAPI/diagnostics-cpp.test | 2 + clang/test/InstallAPI/linker-symbols.test | 440 ++++++++++++++++++ .../mismatching-objc-class-symbols.test | 269 +++++++++++ clang/test/InstallAPI/symbol-flags.test | 290 ++++++++++++ .../clang-installapi/ClangInstallAPI.cpp | 2 +- llvm/lib/TextAPI/BinaryReader/DylibReader.cpp | 7 +- 9 files changed, 1174 insertions(+), 21 deletions(-) create mode 100644 clang/test/InstallAPI/linker-symbols.test create mode 100644 clang/test/InstallAPI/mismatching-objc-class-symbols.test create mode 100644 clang/test/InstallAPI/symbol-flags.test diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td index f99a5fca64cb..a4c6e630ac5f 100644 --- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td +++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td @@ -26,6 +26,7 @@ def warn_library_hidden_symbol : Warning<"declaration has external linkage, but def warn_header_hidden_symbol : Warning<"symbol exported in dynamic library, but marked hidden in declaration '%0'">, InGroup; def err_header_hidden_symbol : Error<"symbol exported in dynamic library, but marked hidden in declaration '%0'">; def err_header_symbol_missing : Error<"no declaration found for exported symbol '%0' in dynamic library">; +def warn_header_symbol_missing : Warning<"no declaration was found for exported symbol '%0' in dynamic library">, InGroup; def warn_header_availability_mismatch : Warning<"declaration '%0' is marked %select{available|unavailable}1," " but symbol is %select{not |}2exported in dynamic library">, InGroup; def err_header_availability_mismatch : Error<"declaration '%0' is marked %select{available|unavailable}1," diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h index bbfa8711313e..49de24763f1f 100644 --- a/clang/include/clang/InstallAPI/DylibVerifier.h +++ b/clang/include/clang/InstallAPI/DylibVerifier.h @@ -28,7 +28,7 @@ enum class VerificationMode { /// lifetime of InstallAPI. /// As declarations are collected during AST traversal, they are /// compared as symbols against what is available in the binary dylib. -class DylibVerifier { +class DylibVerifier : llvm::MachO::RecordVisitor { private: struct SymbolContext; @@ -72,6 +72,9 @@ public: Result verify(ObjCIVarRecord *R, const FrontendAttrs *FA, const StringRef SuperClass); + // Scan through dylib slices and report any remaining missing exports. + Result verifyRemainingSymbols(); + /// Initialize target for verification. void setTarget(const Target &T); @@ -128,6 +131,14 @@ private: /// Find matching dylib slice for target triple that is being parsed. void assignSlice(const Target &T); + /// Shared implementation for verifying exported symbols in dylib. + void visitSymbolInDylib(const Record &R, SymbolContext &SymCtx); + + void visitGlobal(const GlobalRecord &R) override; + void visitObjCInterface(const ObjCInterfaceRecord &R) override; + void visitObjCCategory(const ObjCCategoryRecord &R) override; + void visitObjCIVar(const ObjCIVarRecord &R, const StringRef Super); + /// Gather annotations for symbol for error reporting. std::string getAnnotatedName(const Record *R, SymbolContext &SymCtx, bool ValidSourceLoc = true); diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp index 24e0d0addf2f..94b8e9cd3233 100644 --- a/clang/lib/InstallAPI/DylibVerifier.cpp +++ b/clang/lib/InstallAPI/DylibVerifier.cpp @@ -66,17 +66,15 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, Annotation += "(tlv) "; // Check if symbol represents only part of a @interface declaration. - const bool IsAnnotatedObjCClass = - ((SymCtx.ObjCIFKind != ObjCIFSymbolKind::None) && - (SymCtx.ObjCIFKind <= ObjCIFSymbolKind::EHType)); - - if (IsAnnotatedObjCClass) { - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::EHType) - Annotation += "Exception Type of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::MetaClass) - Annotation += "Metaclass of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::Class) - Annotation += "Class of "; + switch (SymCtx.ObjCIFKind) { + default: + break; + case ObjCIFSymbolKind::EHType: + return Annotation + "Exception Type of " + PrettyName; + case ObjCIFSymbolKind::MetaClass: + return Annotation + "Metaclass of " + PrettyName; + case ObjCIFSymbolKind::Class: + return Annotation + "Class of " + PrettyName; } // Only print symbol type prefix or leading "_" if there is no source location @@ -90,9 +88,6 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, return Annotation + PrettyName; } - if (IsAnnotatedObjCClass) - return Annotation + PrettyName; - switch (SymCtx.Kind) { case EncodeKind::GlobalSymbol: return Annotation + PrettyName; @@ -332,9 +327,9 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, } if (!DR->isThreadLocalValue() && R->isThreadLocalValue()) { Ctx.emitDiag([&]() { - SymCtx.FA->D->getLocation(), - Ctx.Diag->Report(diag::err_header_symbol_flags_mismatch) - << getAnnotatedName(DR, SymCtx) << R->isThreadLocalValue(); + Ctx.Diag->Report(SymCtx.FA->D->getLocation(), + diag::err_header_symbol_flags_mismatch) + << getAnnotatedName(R, SymCtx) << R->isThreadLocalValue(); }); return false; } @@ -520,5 +515,147 @@ void DylibVerifier::VerifierContext::emitDiag( Report(); } +// The existence of weak-defined RTTI can not always be inferred from the +// header files because they can be generated as part of an implementation +// file. +// InstallAPI doesn't warn about weak-defined RTTI, because this doesn't affect +// static linking and so can be ignored for text-api files. +static bool shouldIgnoreCpp(StringRef Name, bool IsWeakDef) { + return (IsWeakDef && + (Name.starts_with("__ZTI") || Name.starts_with("__ZTS"))); +} +void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) { + // Undefined symbols should not be in InstallAPI generated text-api files. + if (R.isUndefined()) { + updateState(Result::Valid); + return; + } + + // Internal symbols should not be in InstallAPI generated text-api files. + if (R.isInternal()) { + updateState(Result::Valid); + return; + } + + // Allow zippered symbols with potentially mismatching availability + // between macOS and macCatalyst in the final text-api file. + const StringRef SymbolName(SymCtx.SymbolName); + if (const Symbol *Sym = Exports->findSymbol(SymCtx.Kind, SymCtx.SymbolName, + SymCtx.ObjCIFKind)) { + if (Sym->hasArchitecture(Ctx.Target.Arch)) { + updateState(Result::Ignore); + return; + } + } + + if (shouldIgnoreCpp(SymbolName, R.isWeakDefined())) { + updateState(Result::Valid); + return; + } + + // All checks at this point classify as some kind of violation that should be + // reported. + + // Regardless of verification mode, error out on mismatched special linker + // symbols. + if (SymbolName.starts_with("$ld$")) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are hard errors on Pedantic mode. + if (Mode == VerificationMode::Pedantic) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are warnings on ErrorsAndWarnings + // mode. + if (Mode == VerificationMode::ErrorsAndWarnings) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::warn_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Ignore); + return; + } + + // Missing declarations are dropped for ErrorsOnly mode. It is the last + // remaining mode. + updateState(Result::Ignore); + return; +} + +void DylibVerifier::visitGlobal(const GlobalRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SimpleSymbol Sym = parseSymbol(R.getName()); + SymCtx.SymbolName = Sym.Name; + SymCtx.Kind = Sym.Kind; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCIVar(const ObjCIVarRecord &R, + const StringRef Super) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = ObjCIVarRecord::createScopedName(Super, R.getName()); + SymCtx.Kind = EncodeKind::ObjectiveCInstanceVariable; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCInterface(const ObjCInterfaceRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = R.getName(); + SymCtx.ObjCIFKind = assignObjCIFSymbolKind(&R); + if (SymCtx.ObjCIFKind > ObjCIFSymbolKind::EHType) { + if (R.hasExceptionAttribute()) { + SymCtx.Kind = EncodeKind::ObjectiveCClassEHType; + visitSymbolInDylib(R, SymCtx); + } + SymCtx.Kind = EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } else { + SymCtx.Kind = R.hasExceptionAttribute() ? EncodeKind::ObjectiveCClassEHType + : EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } + + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getName()); +} + +void DylibVerifier::visitObjCCategory(const ObjCCategoryRecord &R) { + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getSuperClassName()); +} + +DylibVerifier::Result DylibVerifier::verifyRemainingSymbols() { + if (getState() == Result::NoVerify) + return Result::NoVerify; + assert(!Dylib.empty() && "No binary to verify against"); + + Ctx.DiscoveredFirstError = false; + Ctx.PrintArch = true; + for (std::shared_ptr Slice : Dylib) { + Ctx.Target = Slice->getTarget(); + Ctx.DylibSlice = Slice.get(); + Slice->visit(*this); + } + return getState(); +} + } // namespace installapi } // namespace clang diff --git a/clang/test/InstallAPI/diagnostics-cpp.test b/clang/test/InstallAPI/diagnostics-cpp.test index 658886537507..51cca129ea0a 100644 --- a/clang/test/InstallAPI/diagnostics-cpp.test +++ b/clang/test/InstallAPI/diagnostics-cpp.test @@ -21,6 +21,8 @@ CHECK-NEXT: CPP.h:5:7: error: declaration has external linkage, but symbol has i CHECK-NEXT: CPP.h:6:7: error: dynamic library symbol '(weak-def) Bar::init()' is weak defined, but its declaration is not CHECK-NEXT: int init(); CHECK-NEXT: ^ +CHECK-NEXT: warning: violations found for arm64 +CHECK-NEXT: error: no declaration found for exported symbol 'int foo(unsigned int)' in dynamic library //--- inputs.json.in { diff --git a/clang/test/InstallAPI/linker-symbols.test b/clang/test/InstallAPI/linker-symbols.test new file mode 100644 index 000000000000..1e4ddf9c45d5 --- /dev/null +++ b/clang/test/InstallAPI/linker-symbols.test @@ -0,0 +1,440 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/MagicSymbols.yaml -o %t/MagicSymbols + +; RUN: not clang-installapi -target x86_64-apple-macosx13 \ +; RUN: -install_name \ +; RUN: /System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o %t/output.tbd \ +; RUN: --verify-mode=ErrorsOnly \ +; RUN: --verify-against=%t/MagicSymbols 2>&1 | FileCheck %s + +CHECK: warning: violations found for x86_64 +CHECK: error: no declaration found for exported symbol '$ld$add$os10.4$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$add$os10.5$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.6$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.7$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.5$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.4$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' in dynamic library + +;--- MagicSymbols.h +#ifndef SPECIAL_LINKER_SYMBOLS_H +#define SPECIAL_LINKER_SYMBOLS_H + +extern const int SpecialLinkerSymbolsVersion; + +extern int symbol1; +extern int symbol3; + +#endif // SPECIAL_LINKER_SYMBOLS_H + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/MagicSymbols.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- MagicSymbols.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x1000007 + cpusubtype: 0x3 + filetype: 0x6 + ncmds: 12 + sizeofcmds: 952 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 4096 + fileoff: 0 + filesize: 4096 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xBD8 + size: 0 + offset: 0xBD8 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0xBD8 + size: 4 + offset: 0xBD8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '07000000' + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __DATA + vmaddr: 4096 + vmsize: 4096 + fileoff: 4096 + filesize: 4096 + maxprot: 3 + initprot: 3 + nsects: 2 + flags: 0 + Sections: + - sectname: __data + segname: __DATA + addr: 0x1000 + size: 8 + offset: 0x1000 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 4D00000009030000 + - sectname: __common + segname: __DATA + addr: 0x1008 + size: 8 + offset: 0x0 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 8192 + vmsize: 944 + fileoff: 8192 + filesize: 944 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 8192 + export_size: 376 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 8576 + nsyms: 12 + stroff: 8768 + strsize: 368 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 11 + iundefsym: 11 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 120 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols' + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4478-5555-3144-A106-356C3C9DACA3 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88539136 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 8568 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 8576 + datasize: 0 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 11 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 50 + Name: SpecialLinkerSymbolsVersion + Flags: 0x0 + Address: 0xBD8 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 55 + Name: symbol + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 63 + Name: '3' + Flags: 0x0 + Address: 0x1004 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 68 + Name: '1' + Flags: 0x0 + Address: 0x1000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 73 + Name: '$ld$' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 134 + Name: 'add$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 162 + Name: '4$_symbol2' + Flags: 0x0 + Address: 0x1008 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 167 + Name: '5$_symbol2' + Flags: 0x0 + Address: 0x1009 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 172 + Name: 'hide$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 200 + Name: '6$_symbol1' + Flags: 0x0 + Address: 0x100A + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 205 + Name: '7$_symbol1' + Flags: 0x0 + Address: 0x100B + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 210 + Name: 'weak$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 238 + Name: '5$_symbol3' + Flags: 0x0 + Address: 0x100F + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 243 + Name: '4$_symbol3' + Flags: 0x0 + Address: 0x100E + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 248 + Name: 'install_name$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 362 + Name: '4$/System/Library/Frameworks/A.framework/Versions/A/A' + Flags: 0x0 + Address: 0x100C + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 367 + Name: '5$/System/Library/Frameworks/B.framework/Versions/A/B' + Flags: 0x0 + Address: 0x100D + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4104 + - n_strx: 26 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4105 + - n_strx: 50 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4106 + - n_strx: 75 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4107 + - n_strx: 100 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4108 + - n_strx: 176 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4109 + - n_strx: 252 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4110 + - n_strx: 277 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4111 + - n_strx: 302 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 3032 + - n_strx: 331 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4096 + - n_strx: 340 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4100 + - n_strx: 349 + n_type: 0x1 + n_sect: 0 + n_desc: 256 + n_value: 0 + StringTable: + - ' ' + - '$ld$add$os10.4$_symbol2' + - '$ld$add$os10.5$_symbol2' + - '$ld$hide$os10.6$_symbol1' + - '$ld$hide$os10.7$_symbol1' + - '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' + - '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' + - '$ld$weak$os10.4$_symbol3' + - '$ld$weak$os10.5$_symbol3' + - _SpecialLinkerSymbolsVersion + - _symbol1 + - _symbol3 + - dyld_stub_binder + - '' + - '' +... diff --git a/clang/test/InstallAPI/mismatching-objc-class-symbols.test b/clang/test/InstallAPI/mismatching-objc-class-symbols.test new file mode 100644 index 000000000000..3b4acf1035ac --- /dev/null +++ b/clang/test/InstallAPI/mismatching-objc-class-symbols.test @@ -0,0 +1,269 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json +; RUN: yaml2obj %t/swift-objc-class.yaml -o %t/libswift-objc.dylib + +// Try out dylib that only has 1 symbol for a ObjCClass, with no declarations in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/missing.tbd \ +; RUN: --verify-mode=ErrorsAndWarnings 2>&1 | FileCheck --check-prefix MISSING_DECL %s +; RUN: llvm-readtapi --compare %t/missing.tbd %t/missing-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, +// but a complete ObjCClass decl in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/mismatching.tbd \ +; RUN: --verify-mode=Pedantic -DFULL_DECL 2>&1 | FileCheck --check-prefix MISMATCH_DECL %s +; RUN: llvm-readtapi -compare %t/mismatching.tbd %t/mismatching-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, but is represented in header. +; RUN: clang-installapi -target arm64-apple-macos14 \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/matching.tbd \ +; RUN: --verify-mode=Pedantic \ +; RUN: -DHAS_META_DECL 2>&1 | FileCheck --allow-empty %s + +; MISSING_DECL: violations found for arm64 +; MISSING_DECL-NEXT: warning: no declaration was found for exported symbol 'Metaclass of Suggestion' in dynamic library + +; MISMATCH_DECL: violations found for arm64-apple-macos14 +; MISMATCH_DECL: warning: declaration has external linkage, but dynamic library doesn't have symbol 'Class of Suggestion' + +; CHECK-NOT: error +; CHECK-NOT: warning + + +;--- usr/include/mismatch.h +#if HAS_META_DECL +int metaclass __asm("_OBJC_METACLASS_$_Suggestion"); +#endif + +#if FULL_DECL +@interface Suggestion +@end +#endif + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/usr/include/mismatch.h", + "type" : "public" + } + ], + "version": "3" +} + +;--- missing-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +... + +;--- mismatching-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +exports: + - targets: [ arm64-macos ] + objc-classes: [ Suggestion ] +... + +;--- swift-objc-class.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 13 + sizeofcmds: 752 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x330 + size: 0 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0x330 + size: 1 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '61' + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 16384 + vmsize: 416 + fileoff: 16384 + filesize: 416 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 40 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16432 + nsyms: 2 + stroff: 16464 + strsize: 48 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 1 + iundefsym: 1 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 40 + dylib: + name: 24 + timestamp: 0 + current_version: 0 + compatibility_version: 0 + Content: tmp.dylib + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4443-5555-3144-A142-97179769CBE0 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 917504 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 197656576 + compatibility_version: 19660800 + Content: '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation' + ZeroPadBytes: 3 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16424 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16432 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16512 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 32 + Name: '_OBJC_METACLASS_$_Suggestion' + Flags: 0x0 + Address: 0x330 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 816 + - n_strx: 31 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - '_OBJC_METACLASS_$_Suggestion' + - dyld_stub_binder + FunctionStarts: [ 0x330 ] +... +// Generated from: +// xcrun -sdk macosx clang tmp.c -dynamiclib -install_name tmp.dylib +// tmp.c: +// __attribute__((visibility("default"))) +// const char Meta __asm("_OBJC_METACLASS_$_Suggestion") = 'a'; diff --git a/clang/test/InstallAPI/symbol-flags.test b/clang/test/InstallAPI/symbol-flags.test new file mode 100644 index 000000000000..3f68afd17e3b --- /dev/null +++ b/clang/test/InstallAPI/symbol-flags.test @@ -0,0 +1,290 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/flags.yaml -o %t/SymbolFlags + +; RUN: not clang-installapi -x c++ --target=arm64-apple-macos13 \ +; RUN: -install_name /System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o output.tbd \ +; RUN: --verify-against=%t/SymbolFlags \ +; RUN: --verify-mode=ErrorsOnly 2>&1 | FileCheck %s + +; CHECK: project.h:2:21: error: declaration '(tlv) val' is thread local, but symbol is not in dynamic library +; CHECK-NEXT: extern __thread int val; +; CHECK: project.h:3:13: error: dynamic library symbol '(weak-def) __Z12my_weak_funcv' is weak defined, but its declaration is not +; CHECK-NEXT: extern void my_weak_func(); + +;--- project.h +extern void my_func(); +extern __thread int val; +extern void my_weak_func(); + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/project.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- flags.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 14 + sizeofcmds: 912 + flags: 0x118085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xFB0 + size: 8 + offset: 0xFB0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: C0035FD6C0035FD6 + - sectname: __unwind_info + segname: __TEXT + addr: 0xFB8 + size: 4152 + offset: 0xFB8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 010000001C000000010000002000000000000000200000000200000000000002B00F00003800000038000000B80F00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + - cmd: LC_SEGMENT_64 + cmdsize: 152 + segname: __DATA + vmaddr: 16384 + vmsize: 16384 + fileoff: 16384 + filesize: 0 + maxprot: 3 + initprot: 3 + nsects: 1 + flags: 0 + Sections: + - sectname: __common + segname: __DATA + addr: 0x4000 + size: 4 + offset: 0x0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 32768 + vmsize: 480 + fileoff: 16384 + filesize: 480 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 64 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16456 + nsyms: 4 + stroff: 16520 + strsize: 56 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 3 + iundefsym: 3 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags' + ZeroPadBytes: 1 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4436-5555-3144-A1AF-5D3063ACFC99 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 48 + dylib: + name: 24 + timestamp: 0 + current_version: 117985024 + compatibility_version: 65536 + Content: '/usr/lib/libc++.1.dylib' + ZeroPadBytes: 1 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16448 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16456 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16576 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 5 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 4 + NodeOffset: 16 + Name: val + Flags: 0x0 + Address: 0x4000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 22 + Name: _Z + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 52 + Name: 7my_funcv + Flags: 0x0 + Address: 0xFB0 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 57 + Name: 12my_weak_funcv + Flags: 0x4 + Address: 0xFB4 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 1 + n_desc: 0 + n_value: 4016 + - n_strx: 15 + n_type: 0xF + n_sect: 1 + n_desc: 128 + n_value: 4020 + - n_strx: 34 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 16384 + - n_strx: 39 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - __Z7my_funcv + - __Z12my_weak_funcv + - _val + - dyld_stub_binder + FunctionStarts: [ 0xFB0, 0xFB4 ] +... + +/// Generated from: +// clang++ -mtargetos=macosx13 -arch arm64 flags.cpp +// flags.cpp: +// __attribute__((visibility("default"))) void my_func() {} +// __attribute__((weak)) void my_weak_func() {} +// int val = 0; diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp index 54e82d78d4d2..13061cfa36ee 100644 --- a/clang/tools/clang-installapi/ClangInstallAPI.cpp +++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp @@ -123,7 +123,7 @@ static bool run(ArrayRef Args, const char *ProgName) { } } - if (Ctx.Verifier->getState() == DylibVerifier::Result::Invalid) + if (Ctx.Verifier->verifyRemainingSymbols() == DylibVerifier::Result::Invalid) return EXIT_FAILURE; // After symbols have been collected, prepare to write output. diff --git a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp index 0694d8f28df6..2e36d4a8b98c 100644 --- a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp +++ b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp @@ -293,8 +293,11 @@ static Error readSymbols(MachOObjectFile *Obj, RecordsSlice &Slice, RecordLinkage Linkage = RecordLinkage::Unknown; SymbolFlags RecordFlags = SymbolFlags::None; - if (Opt.Undefineds && (Flags & SymbolRef::SF_Undefined)) { - Linkage = RecordLinkage::Undefined; + if (Flags & SymbolRef::SF_Undefined) { + if (Opt.Undefineds) + Linkage = RecordLinkage::Undefined; + else + continue; if (Flags & SymbolRef::SF_Weak) RecordFlags |= SymbolFlags::WeakReferenced; } else if (Flags & SymbolRef::SF_Exported) { -- GitLab From b8e53630f899ddb8a2ec0d37bcb86608d58c4960 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:04:19 -0700 Subject: [PATCH 107/933] [compiler-rt] Avoid pulling in __cxa_pure_virtual When building optimized versions of the runtime libraries the compiler is generally able to elide these references, but when building them for maximum debug info (with -O0), these references remain which causes the test suite to fail for tests that do not pull in the C++ standard library. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/84613 --- .../sanitizer_stacktrace_printer.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h index 10361a320344..e39cb891575e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -30,10 +30,15 @@ class StackTracePrinter { virtual void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, - bool vs_style, - const char *strip_path_prefix = "") = 0; + bool vs_style, const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } - virtual bool RenderNeedsSymbolization(const char *format) = 0; + virtual bool RenderNeedsSymbolization(const char *format) { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, @@ -44,7 +49,10 @@ class StackTracePrinter { const char *strip_path_prefix); virtual void RenderData(InternalScopedString *buffer, const char *format, const DataInfo *DI, - const char *strip_path_prefix = "") = 0; + const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } private: // To be called from StackTracePrinter::GetOrInit -- GitLab From c56211b2430cf63ba3a469a4ae89cf2e829e9332 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Thu, 21 Mar 2024 15:11:31 -0700 Subject: [PATCH 108/933] [libc] Make math-macros.h C++-friendly (#86206) The isfinite, isnan, and isinf "functions" are specified by C99..C23 to be macros that act as type-generic functions. Defining them as their __builtin_* counterparts works fine for this. However, in C++ the identifiers need to be usable in different contexts, such as being declared inside a C++ namespace. So define inline constexpr template functions for them under `#ifdef __cplusplus`. --- libc/include/llvm-libc-macros/math-macros.h | 32 ++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h index db8a4ea65bd6..2605535b927d 100644 --- a/libc/include/llvm-libc-macros/math-macros.h +++ b/libc/include/llvm-libc-macros/math-macros.h @@ -30,10 +30,6 @@ #define FP_LLOGB0 (-LONG_MAX - 1) #define FP_LLOGBNAN LONG_MAX -#define isfinite(x) __builtin_isfinite(x) -#define isinf(x) __builtin_isinf(x) -#define isnan(x) __builtin_isnan(x) - #ifdef __FAST_MATH__ #define math_errhandling 0 #elif defined(__NO_MATH_ERRNO__) @@ -44,4 +40,32 @@ #define math_errhandling (MATH_ERRNO | MATH_ERREXCEPT) #endif +// These must be type-generic functions. The C standard specifies them as +// being macros rather than functions, in fact. However, in C++ it's important +// that there be function declarations that don't interfere with other uses of +// the identifier, even in places with parentheses where a function-like macro +// will be expanded (such as a function declaration in a C++ namespace). + +#ifdef __cplusplus + +template inline constexpr bool isfinite(T x) { + return __builtin_isfinite(x); +} + +template inline constexpr bool isinf(T x) { + return __builtin_isinf(x); +} + +template inline constexpr bool isnan(T x) { + return __builtin_isnan(x); +} + +#else + +#define isfinite(x) __builtin_isfinite(x) +#define isinf(x) __builtin_isinf(x) +#define isnan(x) __builtin_isnan(x) + +#endif + #endif // LLVM_LIBC_MACROS_MATH_MACROS_H -- GitLab From 00f3454bbe04ae8cf0eeda981c439e7f97390bd4 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Thu, 21 Mar 2024 15:12:31 -0700 Subject: [PATCH 109/933] [flang][runtime] Added pseudo file unit for simplified PRINT. (#86134) A file unit is emulated via a temporary buffer that accumulates the output, which is printed out via std::printf at the end of the IO statement. This implementation will be used for the offload devices. --- flang/runtime/CMakeLists.txt | 2 + flang/runtime/external-unit.cpp | 333 ++++++++++++++++++++++++++++++++ flang/runtime/io-stmt.cpp | 10 + flang/runtime/lock.h | 17 +- flang/runtime/pseudo-unit.cpp | 167 ++++++++++++++++ flang/runtime/tools.h | 21 ++ flang/runtime/unit.cpp | 319 +----------------------------- flang/runtime/unit.h | 61 +++++- 8 files changed, 615 insertions(+), 315 deletions(-) create mode 100644 flang/runtime/external-unit.cpp create mode 100644 flang/runtime/pseudo-unit.cpp diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt index 7dd60b5edcd5..021474871154 100644 --- a/flang/runtime/CMakeLists.txt +++ b/flang/runtime/CMakeLists.txt @@ -129,6 +129,7 @@ set(sources exceptions.cpp execute.cpp extensions.cpp + external-unit.cpp extrema.cpp file.cpp findloc.cpp @@ -149,6 +150,7 @@ set(sources numeric.cpp pointer.cpp product.cpp + pseudo-unit.cpp ragged.cpp random.cpp reduction.cpp diff --git a/flang/runtime/external-unit.cpp b/flang/runtime/external-unit.cpp new file mode 100644 index 000000000000..9d650ceca4a8 --- /dev/null +++ b/flang/runtime/external-unit.cpp @@ -0,0 +1,333 @@ +//===-- runtime/external-unit.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit for RT_USE_PSEUDO_FILE_UNIT=0. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "lock.h" +#include "unit-map.h" +#include "unit.h" +#include +#include + +namespace Fortran::runtime::io { + +// The per-unit data structures are created on demand so that Fortran I/O +// should work without a Fortran main program. +static Lock unitMapLock; +static Lock createOpenLock; +static UnitMap *unitMap{nullptr}; + +void FlushOutputOnCrash(const Terminator &terminator) { + if (!defaultOutput && !errorOutput) { + return; + } + IoErrorHandler handler{terminator}; + handler.HasIoStat(); // prevent nested crash if flush has error + CriticalSection critical{unitMapLock}; + if (defaultOutput) { + defaultOutput->FlushOutput(handler); + } + if (errorOutput) { + errorOutput->FlushOutput(handler); + } +} + +ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { + return GetUnitMap().LookUp(unit); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int unit, const Terminator &terminator, bool &wasExtant) { + return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction dir, Fortran::common::optional isUnformatted, + const Terminator &terminator) { + // Make sure that the returned anonymous unit has been opened + // not just created in the unitMap. + CriticalSection critical{createOpenLock}; + bool exists{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; + if (result && !exists) { + IoErrorHandler handler{terminator}; + result->OpenAnonymousUnit( + dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, + Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); + result->isUnformatted = isUnformatted; + } + return result; +} + +ExternalFileUnit *ExternalFileUnit::LookUp( + const char *path, std::size_t pathLen) { + return GetUnitMap().LookUp(path, pathLen); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew( + int unit, const Terminator &terminator) { + bool wasExtant{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, result && !wasExtant); + return *result; +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { + return GetUnitMap().LookUpForClose(unit); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit( + const Terminator &terminator, bool forChildIo) { + ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; + unit.createdForInternalChildIo_ = forChildIo; + return unit; +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional action, Position position, + OwningPtr &&newPath, std::size_t newPathLength, Convert convert, + IoErrorHandler &handler) { + if (convert == Convert::Unknown) { + convert = executionEnvironment.conversion; + } + swapEndianness_ = convert == Convert::Swap || + (convert == Convert::LittleEndian && !isHostLittleEndian) || + (convert == Convert::BigEndian && isHostLittleEndian); + bool impliedClose{false}; + if (IsConnected()) { + bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && + std::memcmp(path(), newPath.get(), newPathLength) == 0}; + if (status && *status != OpenStatus::Old && isSamePath) { + handler.SignalError("OPEN statement for connected unit may not have " + "explicit STATUS= other than 'OLD'"); + return impliedClose; + } + if (!newPath.get() || isSamePath) { + // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= + newPath.reset(); + return impliedClose; + } + // Otherwise, OPEN on open unit with new FILE= implies CLOSE + DoImpliedEndfile(handler); + FlushOutput(handler); + TruncateFrame(0, handler); + Close(CloseStatus::Keep, handler); + impliedClose = true; + } + if (newPath.get() && newPathLength > 0) { + if (const auto *already{ + GetUnitMap().LookUp(newPath.get(), newPathLength)}) { + handler.SignalError(IostatOpenAlreadyConnected, + "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", + unitNumber_, static_cast(newPathLength), newPath.get(), + already->unitNumber_); + return impliedClose; + } + } + set_path(std::move(newPath), newPathLength); + Open(status.value_or(OpenStatus::Unknown), action, position, handler); + auto totalBytes{knownSize()}; + if (access == Access::Direct) { + if (!openRecl) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", + unitNumber()); + } else if (*openRecl <= 0) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", + unitNumber(), static_cast(*openRecl)); + } else if (totalBytes && (*totalBytes % *openRecl != 0)) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " + "even divisor of the file size %jd", + unitNumber(), static_cast(*openRecl), + static_cast(*totalBytes)); + } + recordLength = openRecl; + } + endfileRecordNumber.reset(); + currentRecordNumber = 1; + if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { + endfileRecordNumber = 1 + (*totalBytes / *openRecl); + } + if (position == Position::Append) { + if (totalBytes) { + frameOffsetInFile_ = *totalBytes; + } + if (access != Access::Stream) { + if (!endfileRecordNumber) { + // Fake it so that we can backspace relative from the end + endfileRecordNumber = std::numeric_limits::max() - 2; + } + currentRecordNumber = *endfileRecordNumber; + } + } + return impliedClose; +} + +void ExternalFileUnit::OpenAnonymousUnit( + Fortran::common::optional status, + Fortran::common::optional action, Position position, + Convert convert, IoErrorHandler &handler) { + // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 + std::size_t pathMaxLen{32}; + auto path{SizedNew{handler}(pathMaxLen)}; + std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); + OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), + convert, handler); +} + +void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { + DoImpliedEndfile(handler); + FlushOutput(handler); + Close(status, handler); +} + +void ExternalFileUnit::DestroyClosed() { + GetUnitMap().DestroyClosed(*this); // destroys *this +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction == Direction::Input) { + if (mayRead()) { + direction_ = Direction::Input; + return IostatOk; + } else { + return IostatReadFromWriteOnly; + } + } else { + if (mayWrite()) { + direction_ = Direction::Output; + return IostatOk; + } else { + return IostatWriteToReadOnly; + } + } +} + +UnitMap &ExternalFileUnit::CreateUnitMap() { + Terminator terminator{__FILE__, __LINE__}; + IoErrorHandler handler{terminator}; + UnitMap &newUnitMap{*New{terminator}().release()}; + + bool wasExtant{false}; + ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + out.Predefine(1); + handler.SignalError(out.SetDirection(Direction::Output)); + out.isUnformatted = false; + defaultOutput = &out; + + ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + in.Predefine(0); + handler.SignalError(in.SetDirection(Direction::Input)); + in.isUnformatted = false; + defaultInput = ∈ + + ExternalFileUnit &error{ + *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + error.Predefine(2); + handler.SignalError(error.SetDirection(Direction::Output)); + error.isUnformatted = false; + errorOutput = &error; + + return newUnitMap; +} + +// A back-up atexit() handler for programs that don't terminate with a main +// program END or a STOP statement or other Fortran-initiated program shutdown, +// such as programs with a C main() that terminate normally. It flushes all +// external I/O units. It is registered once the first time that any external +// I/O is attempted. +static void CloseAllExternalUnits() { + IoErrorHandler handler{"Fortran program termination"}; + ExternalFileUnit::CloseAll(handler); +} + +UnitMap &ExternalFileUnit::GetUnitMap() { + if (unitMap) { + return *unitMap; + } + { + CriticalSection critical{unitMapLock}; + if (unitMap) { + return *unitMap; + } + unitMap = &CreateUnitMap(); + } + std::atexit(CloseAllExternalUnits); + return *unitMap; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->CloseAll(handler); + FreeMemoryAndNullify(unitMap); + } + defaultOutput = nullptr; + defaultInput = nullptr; + errorOutput = nullptr; +} + +void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->FlushAll(handler); + } +} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + if (!mayAsynchronous()) { + handler.SignalError(IostatBadAsynchronous); + return -1; + } else { + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + if (auto least{asyncIdAvailable_[j].LeastElement()}) { + asyncIdAvailable_[j].reset(*least); + return 64 * j + static_cast(*least); + } + } + handler.SignalError(IostatTooManyAsyncOps); + return -1; + } +} + +bool ExternalFileUnit::Wait(int id) { + if (static_cast(id) >= maxAsyncIds || + asyncIdAvailable_[id / 64].test(id % 64)) { + return false; + } else { + if (id == 0) { // means "all IDs" + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + asyncIdAvailable_[j].set(); + } + asyncIdAvailable_[0].reset(0); + } else { + asyncIdAvailable_[id / 64].set(id % 64); + } + return true; + } +} + +} // namespace Fortran::runtime::io + +#endif // !defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp index 075d7b5ae518..e3f1214324d8 100644 --- a/flang/runtime/io-stmt.cpp +++ b/flang/runtime/io-stmt.cpp @@ -227,7 +227,17 @@ ConnectionState &ExternalIoStatementBase::GetConnectionState() { return unit_; } int ExternalIoStatementBase::EndIoStatement() { CompleteOperation(); auto result{IoStatementBase::EndIoStatement()}; +#if !defined(RT_USE_PSEUDO_FILE_UNIT) unit_.EndIoStatement(); // annihilates *this in unit_.u_ +#else + // Fetch the unit pointer before *this disappears. + ExternalFileUnit *unitPtr{&unit_}; + // The pseudo file units are dynamically allocated + // and are not tracked in the unit map. + // They have to be destructed and deallocated here. + unitPtr->~ExternalFileUnit(); + FreeMemory(unitPtr); +#endif return result; } diff --git a/flang/runtime/lock.h b/flang/runtime/lock.h index 5fdcf4745c21..61b06a62ff7c 100644 --- a/flang/runtime/lock.h +++ b/flang/runtime/lock.h @@ -12,6 +12,7 @@ #define FORTRAN_RUNTIME_LOCK_H_ #include "terminator.h" +#include "tools.h" // Avoid if possible to avoid introduction of C++ runtime // library dependence. @@ -35,7 +36,17 @@ namespace Fortran::runtime { class Lock { public: -#if USE_PTHREADS +#if RT_USE_PSEUDO_LOCK + // No lock implementation, e.g. for using together + // with RT_USE_PSEUDO_FILE_UNIT. + // The users of Lock class may use it under + // USE_PTHREADS and otherwise, so it has to provide + // all the interfaces. + void Take() {} + bool Try() { return true; } + void Drop() {} + bool TakeIfNoDeadlock() { return true; } +#elif USE_PTHREADS Lock() { pthread_mutex_init(&mutex_, nullptr); } ~Lock() { pthread_mutex_destroy(&mutex_); } void Take() { @@ -79,7 +90,9 @@ public: } private: -#if USE_PTHREADS +#if RT_USE_PSEUDO_FILE_UNIT + // No state. +#elif USE_PTHREADS pthread_mutex_t mutex_{}; volatile bool isBusy_{false}; volatile pthread_t holder_; diff --git a/flang/runtime/pseudo-unit.cpp b/flang/runtime/pseudo-unit.cpp new file mode 100644 index 000000000000..8b5f36e2233a --- /dev/null +++ b/flang/runtime/pseudo-unit.cpp @@ -0,0 +1,167 @@ +//===-- runtime/pseudo-unit.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit and PseudoOpenFile for +// RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "unit.h" +#include + +namespace Fortran::runtime::io { + +void FlushOutputOnCrash(const Terminator &) {} + +ExternalFileUnit *ExternalFileUnit::LookUp(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int, const Terminator &, bool &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction direction, Fortran::common::optional, + const Terminator &terminator) { + if (direction != Direction::Output) { + terminator.Crash("ExternalFileUnit only supports output IO"); + } + return New{terminator}(unit).release(); +} + +ExternalFileUnit *ExternalFileUnit::LookUp(const char *, std::size_t) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew(int, const Terminator &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit(const Terminator &, bool) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional, Position, OwningPtr &&, + std::size_t, Convert, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::OpenAnonymousUnit(Fortran::common::optional, + Fortran::common::optional, Position, Convert convert, + IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::CloseUnit(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::DestroyClosed() { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction != Direction::Output) { + return IostatReadFromWriteOnly; + } + direction_ = direction; + return IostatOk; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &) {} + +void ExternalFileUnit::FlushAll(IoErrorHandler &) {} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::Wait(int) { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::set_mayAsynchronous(bool yes) { + if (yes) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } +} + +Fortran::common::optional +PseudoOpenFile::knownSize() const { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::Open(OpenStatus, Fortran::common::optional, + Position, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Close(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Read( + FileOffset, char *, std::size_t, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Write(FileOffset at, const char *buffer, + std::size_t bytes, IoErrorHandler &handler) { + if (at) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } + // TODO: use persistent string buffer that can be reallocated + // as needed, and only freed at destruction of *this. + auto string{SizedNew{handler}(bytes + 1)}; + std::memcpy(string.get(), buffer, bytes); + string.get()[bytes] = '\0'; + std::printf("%s", string.get()); + return bytes; +} + +void PseudoOpenFile::Truncate(FileOffset, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::ReadAsynchronously( + FileOffset, char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Wait(int, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::WaitAll(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Position PseudoOpenFile::InquirePosition() const { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +} // namespace Fortran::runtime::io + +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h index df25eb888233..c70a1b438e33 100644 --- a/flang/runtime/tools.h +++ b/flang/runtime/tools.h @@ -21,6 +21,27 @@ #include #include +/// \macro RT_PRETTY_FUNCTION +/// Gets a user-friendly looking function signature for the current scope +/// using the best available method on each platform. The exact format of the +/// resulting string is implementation specific and non-portable, so this should +/// only be used, for example, for logging or diagnostics. +/// Copy of LLVM_PRETTY_FUNCTION +#if defined(_MSC_VER) +#define RT_PRETTY_FUNCTION __FUNCSIG__ +#elif defined(__GNUC__) || defined(__clang__) +#define RT_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define RT_PRETTY_FUNCTION __func__ +#endif + +#if defined(RT_DEVICE_COMPILATION) +// Use the pseudo lock and pseudo file unit implementations +// for the device. +#define RT_USE_PSEUDO_LOCK 1 +#define RT_USE_PSEUDO_FILE_UNIT 1 +#endif + namespace Fortran::runtime { class Terminator; diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp index 82f0e68cc20a..67f4775ae0a9 100644 --- a/flang/runtime/unit.cpp +++ b/flang/runtime/unit.cpp @@ -5,293 +5,23 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - +// +// Implementation of ExternalFileUnit common for both +// RT_USE_PSEUDO_FILE_UNIT=0 and RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// #include "unit.h" #include "io-error.h" #include "lock.h" #include "tools.h" -#include "unit-map.h" -#include "flang/Runtime/magic-numbers.h" -#include #include #include namespace Fortran::runtime::io { -// The per-unit data structures are created on demand so that Fortran I/O -// should work without a Fortran main program. -static Lock unitMapLock; -static Lock createOpenLock; -static UnitMap *unitMap{nullptr}; -static ExternalFileUnit *defaultInput{nullptr}; // unit 5 -static ExternalFileUnit *defaultOutput{nullptr}; // unit 6 -static ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension - -void FlushOutputOnCrash(const Terminator &terminator) { - if (!defaultOutput && !errorOutput) { - return; - } - IoErrorHandler handler{terminator}; - handler.HasIoStat(); // prevent nested crash if flush has error - CriticalSection critical{unitMapLock}; - if (defaultOutput) { - defaultOutput->FlushOutput(handler); - } - if (errorOutput) { - errorOutput->FlushOutput(handler); - } -} - -ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { - return GetUnitMap().LookUp(unit); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( - int unit, const Terminator &terminator, bool &wasExtant) { - return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, - Direction dir, Fortran::common::optional isUnformatted, - const Terminator &terminator) { - // Make sure that the returned anonymous unit has been opened - // not just created in the unitMap. - CriticalSection critical{createOpenLock}; - bool exists{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; - if (result && !exists) { - IoErrorHandler handler{terminator}; - result->OpenAnonymousUnit( - dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, - Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); - result->isUnformatted = isUnformatted; - } - return result; -} - -ExternalFileUnit *ExternalFileUnit::LookUp( - const char *path, std::size_t pathLen) { - return GetUnitMap().LookUp(path, pathLen); -} - -ExternalFileUnit &ExternalFileUnit::CreateNew( - int unit, const Terminator &terminator) { - bool wasExtant{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, result && !wasExtant); - return *result; -} - -ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { - return GetUnitMap().LookUpForClose(unit); -} - -ExternalFileUnit &ExternalFileUnit::NewUnit( - const Terminator &terminator, bool forChildIo) { - ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; - unit.createdForInternalChildIo_ = forChildIo; - return unit; -} - -bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, - Fortran::common::optional action, Position position, - OwningPtr &&newPath, std::size_t newPathLength, Convert convert, - IoErrorHandler &handler) { - if (convert == Convert::Unknown) { - convert = executionEnvironment.conversion; - } - swapEndianness_ = convert == Convert::Swap || - (convert == Convert::LittleEndian && !isHostLittleEndian) || - (convert == Convert::BigEndian && isHostLittleEndian); - bool impliedClose{false}; - if (IsConnected()) { - bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && - std::memcmp(path(), newPath.get(), newPathLength) == 0}; - if (status && *status != OpenStatus::Old && isSamePath) { - handler.SignalError("OPEN statement for connected unit may not have " - "explicit STATUS= other than 'OLD'"); - return impliedClose; - } - if (!newPath.get() || isSamePath) { - // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= - newPath.reset(); - return impliedClose; - } - // Otherwise, OPEN on open unit with new FILE= implies CLOSE - DoImpliedEndfile(handler); - FlushOutput(handler); - TruncateFrame(0, handler); - Close(CloseStatus::Keep, handler); - impliedClose = true; - } - if (newPath.get() && newPathLength > 0) { - if (const auto *already{ - GetUnitMap().LookUp(newPath.get(), newPathLength)}) { - handler.SignalError(IostatOpenAlreadyConnected, - "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", - unitNumber_, static_cast(newPathLength), newPath.get(), - already->unitNumber_); - return impliedClose; - } - } - set_path(std::move(newPath), newPathLength); - Open(status.value_or(OpenStatus::Unknown), action, position, handler); - auto totalBytes{knownSize()}; - if (access == Access::Direct) { - if (!openRecl) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", - unitNumber()); - } else if (*openRecl <= 0) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", - unitNumber(), static_cast(*openRecl)); - } else if (totalBytes && (*totalBytes % *openRecl != 0)) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " - "even divisor of the file size %jd", - unitNumber(), static_cast(*openRecl), - static_cast(*totalBytes)); - } - recordLength = openRecl; - } - endfileRecordNumber.reset(); - currentRecordNumber = 1; - if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { - endfileRecordNumber = 1 + (*totalBytes / *openRecl); - } - if (position == Position::Append) { - if (totalBytes) { - frameOffsetInFile_ = *totalBytes; - } - if (access != Access::Stream) { - if (!endfileRecordNumber) { - // Fake it so that we can backspace relative from the end - endfileRecordNumber = std::numeric_limits::max() - 2; - } - currentRecordNumber = *endfileRecordNumber; - } - } - return impliedClose; -} - -void ExternalFileUnit::OpenAnonymousUnit( - Fortran::common::optional status, - Fortran::common::optional action, Position position, - Convert convert, IoErrorHandler &handler) { - // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 - std::size_t pathMaxLen{32}; - auto path{SizedNew{handler}(pathMaxLen)}; - std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); - OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), - convert, handler); -} - -void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { - DoImpliedEndfile(handler); - FlushOutput(handler); - Close(status, handler); -} - -void ExternalFileUnit::DestroyClosed() { - GetUnitMap().DestroyClosed(*this); // destroys *this -} - -Iostat ExternalFileUnit::SetDirection(Direction direction) { - if (direction == Direction::Input) { - if (mayRead()) { - direction_ = Direction::Input; - return IostatOk; - } else { - return IostatReadFromWriteOnly; - } - } else { - if (mayWrite()) { - direction_ = Direction::Output; - return IostatOk; - } else { - return IostatWriteToReadOnly; - } - } -} - -UnitMap &ExternalFileUnit::CreateUnitMap() { - Terminator terminator{__FILE__, __LINE__}; - IoErrorHandler handler{terminator}; - UnitMap &newUnitMap{*New{terminator}().release()}; - - bool wasExtant{false}; - ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - out.Predefine(1); - handler.SignalError(out.SetDirection(Direction::Output)); - out.isUnformatted = false; - defaultOutput = &out; - - ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - in.Predefine(0); - handler.SignalError(in.SetDirection(Direction::Input)); - in.isUnformatted = false; - defaultInput = ∈ - - ExternalFileUnit &error{ - *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - error.Predefine(2); - handler.SignalError(error.SetDirection(Direction::Output)); - error.isUnformatted = false; - errorOutput = &error; - - return newUnitMap; -} - -// A back-up atexit() handler for programs that don't terminate with a main -// program END or a STOP statement or other Fortran-initiated program shutdown, -// such as programs with a C main() that terminate normally. It flushes all -// external I/O units. It is registered once the first time that any external -// I/O is attempted. -static void CloseAllExternalUnits() { - IoErrorHandler handler{"Fortran program termination"}; - ExternalFileUnit::CloseAll(handler); -} - -UnitMap &ExternalFileUnit::GetUnitMap() { - if (unitMap) { - return *unitMap; - } - { - CriticalSection critical{unitMapLock}; - if (unitMap) { - return *unitMap; - } - unitMap = &CreateUnitMap(); - } - std::atexit(CloseAllExternalUnits); - return *unitMap; -} - -void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->CloseAll(handler); - FreeMemoryAndNullify(unitMap); - } - defaultOutput = nullptr; - defaultInput = nullptr; - errorOutput = nullptr; -} - -void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->FlushAll(handler); - } -} +ExternalFileUnit *defaultInput{nullptr}; // unit 5 +ExternalFileUnit *defaultOutput{nullptr}; // unit 6 +ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension static inline void SwapEndianness( char *data, std::size_t bytes, std::size_t elementBytes) { @@ -999,39 +729,6 @@ void ExternalFileUnit::PopChildIo(ChildIo &child) { child_.reset(child.AcquirePrevious().release()); // deletes top child } -int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { - if (!mayAsynchronous()) { - handler.SignalError(IostatBadAsynchronous); - return -1; - } else { - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - if (auto least{asyncIdAvailable_[j].LeastElement()}) { - asyncIdAvailable_[j].reset(*least); - return 64 * j + static_cast(*least); - } - } - handler.SignalError(IostatTooManyAsyncOps); - return -1; - } -} - -bool ExternalFileUnit::Wait(int id) { - if (static_cast(id) >= maxAsyncIds || - asyncIdAvailable_[id / 64].test(id % 64)) { - return false; - } else { - if (id == 0) { // means "all IDs" - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - asyncIdAvailable_[j].set(); - } - asyncIdAvailable_[0].reset(0); - } else { - asyncIdAvailable_[id / 64].set(id % 64); - } - return true; - } -} - std::int32_t ExternalFileUnit::ReadHeaderOrFooter(std::int64_t frameOffset) { std::int32_t word; char *wordPtr{reinterpret_cast(&word)}; diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h index fc5bead7e1d9..5f854abd42f6 100644 --- a/flang/runtime/unit.h +++ b/flang/runtime/unit.h @@ -31,10 +31,67 @@ namespace Fortran::runtime::io { class UnitMap; class ChildIo; +class ExternalFileUnit; + +// Predefined file units. +extern ExternalFileUnit *defaultInput; // unit 5 +extern ExternalFileUnit *defaultOutput; // unit 6 +extern ExternalFileUnit *errorOutput; // unit 0 extension + +#if defined(RT_USE_PSEUDO_FILE_UNIT) +// A flavor of OpenFile class that pretends to be a terminal, +// and only provides basic buffering of the output +// in an internal buffer, and Write's the output +// using std::printf(). Since it does not rely on file system +// APIs, it can be used to implement external output +// for offload devices. +class PseudoOpenFile { +public: + using FileOffset = std::int64_t; + + const char *path() const { return nullptr; } + std::size_t pathLength() const { return 0; } + void set_path(OwningPtr &&, std::size_t bytes) {} + bool mayRead() const { return false; } + bool mayWrite() const { return true; } + bool mayPosition() const { return false; } + bool mayAsynchronous() const { return false; } + void set_mayAsynchronous(bool yes); + // Pretend to be a terminal to force the output + // at the end of IO statement. + bool isTerminal() const { return true; } + bool isWindowsTextFile() const { return false; } + Fortran::common::optional knownSize() const; + bool IsConnected() const { return false; } + void Open(OpenStatus, Fortran::common::optional, Position, + IoErrorHandler &); + void Predefine(int fd) {} + void Close(CloseStatus, IoErrorHandler &); + std::size_t Read(FileOffset, char *, std::size_t minBytes, + std::size_t maxBytes, IoErrorHandler &); + std::size_t Write(FileOffset, const char *, std::size_t, IoErrorHandler &); + void Truncate(FileOffset, IoErrorHandler &); + int ReadAsynchronously(FileOffset, char *, std::size_t, IoErrorHandler &); + int WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &); + void Wait(int id, IoErrorHandler &); + void WaitAll(IoErrorHandler &); + Position InquirePosition() const; +}; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = OpenFile; +using FileFrameClass = FileFrame; +#else // defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = PseudoOpenFile; +// Use not so big buffer for the pseudo file unit frame. +using FileFrameClass = FileFrame; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) class ExternalFileUnit : public ConnectionState, - public OpenFile, - public FileFrame { + public OpenFileClass, + public FileFrameClass { public: static constexpr int maxAsyncIds{64 * 16}; -- GitLab From 6f9297fc4da9df776aef7ee9a18ac426053aaed4 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:22:06 -0700 Subject: [PATCH 110/933] [compiler-rt] Fix build race with COMPILER_RT_TEST_STANDALONE_BUILD_LIBS Since this standalone build configuration uses the runtime libraries that are being built just now, we need to ensure that e.g. the TSan unit tests depend on the tsan runtime library. Also fix TSAN_DEPS being overridden to not include the tsan runtime (commit .....). This change fixes a build race seen in the CI checks for TsanRtlTest-x86_64-Test in https://github.com/llvm/llvm-project/pull/83088. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83650 --- compiler-rt/cmake/Modules/CompilerRTCompile.cmake | 2 +- compiler-rt/lib/msan/tests/CMakeLists.txt | 6 +++--- compiler-rt/lib/tsan/tests/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake index d5b2e7970f11..1629db18f1c2 100644 --- a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake @@ -46,7 +46,7 @@ function(sanitizer_test_compile obj_list source arch) # Write out architecture-specific flags into TARGET_CFLAGS variable. get_target_flags_for_arch(${arch} TARGET_CFLAGS) set(COMPILE_DEPS ${TEST_COMPILE_DEPS}) - if(NOT COMPILER_RT_STANDALONE_BUILD) + if(NOT COMPILER_RT_STANDALONE_BUILD OR COMPILER_RT_TEST_STANDALONE_BUILD_LIBS) list(APPEND COMPILE_DEPS ${TEST_DEPS}) endif() clang_compile(${output_obj} ${source} diff --git a/compiler-rt/lib/msan/tests/CMakeLists.txt b/compiler-rt/lib/msan/tests/CMakeLists.txt index 4f09f1e6a691..e0771dd5a1a7 100644 --- a/compiler-rt/lib/msan/tests/CMakeLists.txt +++ b/compiler-rt/lib/msan/tests/CMakeLists.txt @@ -69,9 +69,9 @@ macro(msan_compile obj_list source arch kind cflags) sanitizer_test_compile( ${obj_list} ${source} ${arch} KIND ${kind} - COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} + COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} libcxx_msan_${arch}-build DEPS msan - CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1 + CFLAGS -isystem ${MSAN_LIBCXX_DIR}/../include/c++/v1 ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags} ) endmacro() @@ -120,7 +120,7 @@ macro(add_msan_tests_for_arch arch kind cflags) set(MSAN_TEST_DEPS ${MSAN_TEST_OBJECTS} libcxx_msan_${arch}-build ${MSAN_LOADABLE_SO} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a") - list(APPEND MSAN_TEST_DEPS msan) + list(APPEND MSAN_TEST_DEPS msan libcxx_msan_${arch}-build) get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) add_compiler_rt_test(MsanUnitTests "Msan-${arch}${kind}-Test" ${arch} OBJECTS ${MSAN_TEST_OBJECTS} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a" diff --git a/compiler-rt/lib/tsan/tests/CMakeLists.txt b/compiler-rt/lib/tsan/tests/CMakeLists.txt index ad8cc9b0eb05..1bc08bbf7450 100644 --- a/compiler-rt/lib/tsan/tests/CMakeLists.txt +++ b/compiler-rt/lib/tsan/tests/CMakeLists.txt @@ -67,7 +67,7 @@ endforeach() set(TSAN_DEPS tsan) # TSan uses C++ standard library headers. if (TARGET cxx-headers OR HAVE_LIBCXX) - set(TSAN_DEPS cxx-headers) + list(APPEND TSAN_DEPS cxx-headers) endif() # add_tsan_unittest( -- GitLab From 8d1affb87181b9636b87e04a245bcde06f8a7d47 Mon Sep 17 00:00:00 2001 From: Diego Caballero Date: Thu, 21 Mar 2024 15:28:36 -0700 Subject: [PATCH 111/933] Update @dcaballe in CODEOWNERS (#86177) It fixes a few rules that don't seem to be working and adding myself to a few paths where I've been contributing and can offer my review. Also minor sorting changes. --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c246c42b0904..9cf6f5d79d41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -59,8 +59,8 @@ clang/test/AST/Interp/ @tbaederr /mlir/Dialect/*/Transforms/Bufferize.cpp @matthias-springer # Linalg Dialect in MLIR. -/mlir/include/mlir/Dialect/Linalg @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Linalg @dcaballe @nicolasvasilache +/mlir/include/mlir/Dialect/Linalg/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Linalg/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache @@ -77,14 +77,14 @@ clang/test/AST/Interp/ @tbaederr /mlir/**/*SME* @banach-space @dcaballe @nicolasvasilache /mlir/**/*SVE* @banach-space @dcaballe @nicolasvasilache /mlir/**/*VectorInterfaces* @dcaballe @nicolasvasilache -/mlir/**/*VectorToSCF* @banach-space @dcaballe @nicolasvasilache @matthias-springer +/mlir/**/*VectorToSCF* @banach-space @dcaballe @matthias-springer @nicolasvasilache /mlir/**/*VectorToLLVM* @banach-space @dcaballe @nicolasvasilache /mlir/**/*X86Vector* @aartbik @dcaballe @nicolasvasilache -/mlir/include/mlir/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache -/mlir/**/*EmulateNarrowType* @hanhanW +/mlir/include/mlir/Dialect/Vector/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Vector/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Vector/Transforms/* @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/**/*EmulateNarrowType* @dcaballe @hanhanW # Presburger library in MLIR /mlir/**/*Presburger* @Groverkss @Superty -- GitLab From 1538b82fd395a2fba90842b8a8010b8dcc919499 Mon Sep 17 00:00:00 2001 From: Cooper Partin Date: Thu, 21 Mar 2024 15:40:29 -0700 Subject: [PATCH 112/933] Revert "Add support for PSV EntryFunctionName (#84409)" (#86211) This reverts commit cde54df39cab3a1d60a3e1862ab341609bee3cc3. Co-authored-by: Cooper Partin --- llvm/include/llvm/BinaryFormat/DXContainer.h | 13 --- llvm/include/llvm/MC/DXContainerPSVInfo.h | 25 ++-- llvm/include/llvm/MC/StringTableBuilder.h | 8 +- llvm/include/llvm/Object/DXContainer.h | 16 +-- .../include/llvm/ObjectYAML/DXContainerYAML.h | 5 +- llvm/lib/MC/DXContainerPSVInfo.cpp | 75 ++++-------- llvm/lib/Object/DXContainer.cpp | 15 +-- llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 3 +- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 15 --- .../DXContainer/PSVv3-amplification.yaml | 97 ---------------- .../ObjectYAML/DXContainer/PSVv3-compute.yaml | 95 --------------- .../ObjectYAML/DXContainer/PSVv3-domain.yaml | 105 ----------------- .../DXContainer/PSVv3-geometry.yaml | 105 ----------------- .../ObjectYAML/DXContainer/PSVv3-hull.yaml | 107 ----------------- .../ObjectYAML/DXContainer/PSVv3-mesh.yaml | 109 ------------------ .../ObjectYAML/DXContainer/PSVv3-pixel.yaml | 99 ---------------- .../ObjectYAML/DXContainer/PSVv3-vertex.yaml | 97 ---------------- llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 3 - 18 files changed, 52 insertions(+), 940 deletions(-) delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index ba882c4a6f32..532f9481766a 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -424,19 +424,6 @@ struct ResourceBindInfo : public v0::ResourceBindInfo { }; } // namespace v2 - -namespace v3 { -struct RuntimeInfo : public v2::RuntimeInfo { - uint32_t EntryNameOffset; - - void swapBytes() { sys::swapByteOrder(EntryNameOffset); } - - void swapBytes(Triple::EnvironmentType Stage) { - v2::RuntimeInfo::swapBytes(Stage); - } -}; - -} // namespace v3 } // namespace PSV #define COMPONENT_PRECISION(Val, Enum) Enum = Val, diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h index bad2fe78eb8f..7d21c18d252f 100644 --- a/llvm/include/llvm/MC/DXContainerPSVInfo.h +++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h @@ -9,11 +9,9 @@ #ifndef LLVM_MC_DXCONTAINERPSVINFO_H #define LLVM_MC_DXCONTAINERPSVINFO_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" -#include "llvm/MC/StringTableBuilder.h" #include "llvm/TargetParser/Triple.h" #include @@ -47,9 +45,8 @@ struct PSVSignatureElement { // modifiable format, and can be used to serialize the data back into valid PSV // RuntimeInfo. struct PSVRuntimeInfo { - PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {} bool IsFinalized = false; - dxbc::PSV::v3::RuntimeInfo BaseData; + dxbc::PSV::v2::RuntimeInfo BaseData; SmallVector Resources; SmallVector InputElements; SmallVector OutputElements; @@ -67,7 +64,6 @@ struct PSVRuntimeInfo { std::array, 4> InputOutputMap; SmallVector InputPatchMap; SmallVector PatchOutputMap; - llvm::StringRef EntryName; // Serialize PSVInfo into the provided raw_ostream. The version field // specifies the data version to encode, the default value specifies encoding @@ -75,12 +71,19 @@ struct PSVRuntimeInfo { void write(raw_ostream &OS, uint32_t Version = std::numeric_limits::max()) const; - void finalize(Triple::EnvironmentType Stage); - -private: - SmallVector IndexBuffer; - SmallVector SignatureElements; - StringTableBuilder DXConStrTabBuilder; + void finalize(Triple::EnvironmentType Stage) { + IsFinalized = true; + BaseData.SigInputElements = static_cast(InputElements.size()); + BaseData.SigOutputElements = static_cast(OutputElements.size()); + BaseData.SigPatchOrPrimElements = + static_cast(PatchOrPrimElements.size()); + if (!sys::IsBigEndianHost) + return; + BaseData.swapBytes(); + BaseData.swapBytes(Stage); + for (auto &Res : Resources) + Res.swapBytes(); + } }; class Signature { diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index a738683548cf..4ee421e22c17 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -74,8 +74,12 @@ public: /// Check if a string is contained in the string table. Since this class /// doesn't store the string values, this function can be used to check if /// storage needs to be done prior to adding the string. - bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); } - bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); } + bool contains(StringRef S) const { + return contains(CachedHashStringRef(S)); + } + bool contains(CachedHashStringRef S) const { + return StringIndexMap.count(S); + } size_t getSize() const { return Size; } void clear(); diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index 19c83ba6c6e8..b6e3d321da24 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -125,8 +125,7 @@ class PSVRuntimeInfo { uint32_t Size; using InfoStruct = std::variant; + dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo>; InfoStruct BasicInfo; ResourceArray Resources; StringRef StringTable; @@ -152,11 +151,9 @@ public: ResourceArray getResources() const { return Resources; } uint32_t getVersion() const { - return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo) - ? 3 - : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) ? 2 - : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1 - : 0); + return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) + ? 2 + : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0); } uint32_t getResourceStride() const { return Resources.Stride; } @@ -164,11 +161,6 @@ public: const InfoStruct &getInfo() const { return BasicInfo; } template const T *getInfoAs() const { - if (const auto *P = std::get_if(&BasicInfo)) - return static_cast(P); - if (std::is_same::value) - return nullptr; - if (const auto *P = std::get_if(&BasicInfo)) return static_cast(P); if (std::is_same::value) diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index 9c4d9e19f11b..f7f8d5e6bf47 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -107,7 +107,7 @@ struct PSVInfo { // the format. uint32_t Version; - dxbc::PSV::v3::RuntimeInfo Info; + dxbc::PSV::v2::RuntimeInfo Info; uint32_t ResourceStride; SmallVector Resources; SmallVector SigInputElements; @@ -121,15 +121,12 @@ struct PSVInfo { MaskVector InputPatchMap; MaskVector PatchOutputMap; - StringRef EntryName; - void mapInfoForVersion(yaml::IO &IO); PSVInfo(); PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage); PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P); PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P); - PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable); }; struct SignatureParameter { diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp index aeff69380139..48182fcd31df 100644 --- a/llvm/lib/MC/DXContainerPSVInfo.cpp +++ b/llvm/lib/MC/DXContainerPSVInfo.cpp @@ -81,18 +81,13 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo); break; case 2: - InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); - BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); - break; - case 3: default: - InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo); + InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); } - // Write the size of the info. - support::endian::write(OS, InfoSize, llvm::endianness::little); + support::endian::write(OS, InfoSize, llvm::endianness::little); // Write the info itself. OS.write(reinterpret_cast(&BaseData), InfoSize); @@ -109,12 +104,32 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { if (Version == 0) return; - support::endian::write(OS, - static_cast(DXConStrTabBuilder.getSize()), + StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer)); + SmallVector IndexBuffer; + SmallVector SignatureElements; + SmallVector SemanticNames; + + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, InputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, OutputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, PatchOrPrimElements); + + StrTabBuilder.finalize(); + for (auto ElAndName : zip(SignatureElements, SemanticNames)) { + v0::SignatureElement &El = std::get<0>(ElAndName); + StringRef Name = std::get<1>(ElAndName); + El.NameOffset = static_cast(StrTabBuilder.getOffset(Name)); + if (sys::IsBigEndianHost) + El.swapBytes(); + } + + support::endian::write(OS, static_cast(StrTabBuilder.getSize()), llvm::endianness::little); // Write the string table. - DXConStrTabBuilder.write(OS); + StrTabBuilder.write(OS); // Write the index table size, then table. support::endian::write(OS, static_cast(IndexBuffer.size()), @@ -147,46 +162,6 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { llvm::endianness::little); } -void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) { - IsFinalized = true; - BaseData.SigInputElements = static_cast(InputElements.size()); - BaseData.SigOutputElements = static_cast(OutputElements.size()); - BaseData.SigPatchOrPrimElements = - static_cast(PatchOrPrimElements.size()); - - SmallVector SemanticNames; - - // Build a string table and set associated offsets to be written when - // write() is called - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, InputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, OutputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, PatchOrPrimElements); - - DXConStrTabBuilder.add(EntryName); - - DXConStrTabBuilder.finalize(); - for (auto ElAndName : zip(SignatureElements, SemanticNames)) { - llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName); - StringRef Name = std::get<1>(ElAndName); - El.NameOffset = static_cast(DXConStrTabBuilder.getOffset(Name)); - if (sys::IsBigEndianHost) - El.swapBytes(); - } - - BaseData.EntryNameOffset = - static_cast(DXConStrTabBuilder.getOffset(EntryName)); - - if (!sys::IsBigEndianHost) - return; - BaseData.swapBytes(); - BaseData.swapBytes(Stage); - for (auto &Res : Resources) - Res.swapBytes(); -} - void Signature::write(raw_ostream &OS) { SmallVector SigParams; SigParams.reserve(Params.size()); diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 3b1a6203a1f8..935749afe338 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -247,14 +247,7 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { const uint32_t PSVVersion = getVersion(); // Detect the PSVVersion by looking at the size field. - if (PSVVersion == 3) { - v3::RuntimeInfo Info; - if (Error Err = readStruct(PSVInfoData, Current, Info)) - return Err; - if (sys::IsBigEndianHost) - Info.swapBytes(ShaderStage); - BasicInfo = Info; - } else if (PSVVersion == 2) { + if (PSVVersion == 2) { v2::RuntimeInfo Info; if (Error Err = readStruct(PSVInfoData, Current, Info)) return Err; @@ -432,8 +425,6 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { } uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -442,8 +433,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -452,8 +441,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index f3a518df3175..09a5e41c7123 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -198,9 +198,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { if (!P.Info.has_value()) continue; mcdxbc::PSVRuntimeInfo PSV; - memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo)); + memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo)); PSV.Resources = P.Info->Resources; - PSV.EntryName = P.Info->EntryName; for (auto El : P.Info->SigInputElements) PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{ diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index 38063670aee6..a6871e7855e4 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -74,16 +74,6 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P) memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo)); } -DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, - StringRef StringTable) - : Version(3), - EntryName(StringTable.substr(P->EntryNameOffset, - StringTable.find('\0', P->EntryNameOffset) - - P->EntryNameOffset)) { - memset(&Info, 0, sizeof(Info)); - memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo)); -} - namespace yaml { void MappingTraits::mapping( @@ -358,11 +348,6 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) { IO.mapRequired("NumThreadsX", Info.NumThreadsX); IO.mapRequired("NumThreadsY", Info.NumThreadsY); IO.mapRequired("NumThreadsZ", Info.NumThreadsZ); - - if (Version == 2) - return; - - IO.mapRequired("EntryName", EntryName); } } // namespace llvm diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml deleted file mode 100644 index 09885bd529f0..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 14 - PayloadSizeInBytes: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: ASEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 14 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 14 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: ASEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml deleted file mode 100644 index ee6fb112c772..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 5 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: CSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 5 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 5 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: CSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml deleted file mode 100644 index dd367deae88e..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 4 - InputControlPointCount: 1024 - OutputPositionPresent: 1 - TessellatorDomain: 2056 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: DSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - PatchOutputMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 4 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 4 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: DSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: PatchOutputMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml deleted file mode 100644 index 4c7680b63b02..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 2 - InputPrimitive: 1024 - OutputTopology: 4096 - OutputStreamMask: 2056 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - MaxVertexCount: 4096 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: GSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 2 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 2 -# CHECK-NEXT: InputPrimitive: 1024 -# CHECK-NEXT: OutputTopology: 4096 -# CHECK-NEXT: OutputStreamMask: 2056 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: MaxVertexCount: 4096 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: GSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml deleted file mode 100644 index 3bbad8a9b0ee..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 3 - InputControlPointCount: 1024 - OutputControlPointCount: 4096 - TessellatorDomain: 2056 - TessellatorOutputPrimitive: 8192 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: HSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - InputPatchMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 3 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 3 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputControlPointCount: 4096 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: TessellatorOutputPrimitive: 8192 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: HSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: InputPatchMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml deleted file mode 100644 index c5ea1fcf0780..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 13 - GroupSharedBytesUsed: 1024 - GroupSharedBytesDependentOnViewID: 2056 - PayloadSizeInBytes: 4092 - MaxOutputVertices: 8196 - MaxOutputPrimitives: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPrimVectors: 128 - MeshOutputTopology: 16 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: MSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 13 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 13 -# CHECK-NEXT: GroupSharedBytesUsed: 1024 -# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MaxOutputVertices: 8196 -# CHECK-NEXT: MaxOutputPrimitives: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPrimVectors: 128 -# CHECK-NEXT: MeshOutputTopology: 16 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: MSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml deleted file mode 100644 index b28d5ec8074d..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 0 - DepthOutput: 7 - SampleFrequency: 96 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: PSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 0 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 0 -# CHECK-NEXT: DepthOutput: 7 -# CHECK-NEXT: SampleFrequency: 96 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: PSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml deleted file mode 100644 index d1fb55839931..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 1 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: VSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 1 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 1 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: VSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index ec4f5c74498f..69d9b9a2f784 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -99,9 +99,6 @@ dumpDXContainer(MemoryBufferRef Source) { else if (const auto *P = std::get_if(&PSVInfo->getInfo())) NewPart.Info = DXContainerYAML::PSVInfo(P); - else if (const auto *P = - std::get_if(&PSVInfo->getInfo())) - NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable()); NewPart.Info->ResourceStride = PSVInfo->getResourceStride(); for (auto Res : PSVInfo->getResources()) NewPart.Info->Resources.push_back(Res); -- GitLab From dc74bf7a5412df82223f7062d9a6b814abbfca45 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:41:49 -0700 Subject: [PATCH 113/933] Add myself as codeowner for InstallAPI & TextAPI --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9cf6f5d79d41..fea132c8fe78 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -119,3 +119,8 @@ clang/test/AST/Interp/ @tbaederr # Bazel build system. /utils/bazel/ @rupprecht + +# InstallAPI and TextAPI +/llvm/**/TextAPI/ @cyndyishida +/clang/**/InstallAPI/ @cyndyishida +/clang/tools/clang-installapi/ @cyndyishida -- GitLab From 06d245242e3e24cd4558f545fb5ceba0582c4f03 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:08:51 +0800 Subject: [PATCH 114/933] [RISCV] Recursively split concat_vector into smaller LMULs when lowering (#85825) This is a reimplementation of the combine added in #83035 but as a lowering instead of a combine, so we don't regress the test case added in e59f120e3a14ccdc55fcb7be996efaa768daabe0 by interfering with the strided load combine Previously the combine had to concatenate the split vectors with insert_subvector instead of concat_vectors to prevent an infinite combine loop. And the reasoning behind keeping it as a combine was because if we emitted the insert_subvector during lowering then we didn't fold away inserts of undef subvectors. However it turns out we can avoid this if we just do this in lowering and select a concat_vector directly, since we get the undef folding for free with `DAG.getNode(ISD::CONCAT_VECTOR, ...)` via foldCONCAT_VECTORS. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 24 + .../CodeGen/RISCV/rvv/active_lane_mask.ll | 91 +- .../RISCV/rvv/combine-store-extract-crash.ll | 4 +- .../CodeGen/RISCV/rvv/extract-subvector.ll | 3 +- .../RISCV/rvv/fixed-vectors-shuffle-concat.ll | 254 ++-- .../rvv/fixed-vectors-strided-load-combine.ll | 105 +- .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll | 1066 ++++++++++------- llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll | 13 +- llvm/test/CodeGen/RISCV/rvv/pr63596.ll | 37 +- 9 files changed, 844 insertions(+), 753 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index a3ebfb34ad7a..71059b5bdc0f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -6611,6 +6611,30 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, // better than going through the stack, as the default expansion does. SDLoc DL(Op); MVT VT = Op.getSimpleValueType(); + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) + ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget); + + // Recursively split concat_vectors with more than 2 operands: + // + // concat_vector op1, op2, op3, op4 + // -> + // concat_vector (concat_vector op1, op2), (concat_vector op3, op4) + // + // This reduces the length of the chain of vslideups and allows us to + // perform the vslideups at a smaller LMUL, limited to MF2. + if (Op.getNumOperands() > 2 && + ContainerVT.bitsGE(getLMUL1VT(ContainerVT))) { + MVT HalfVT = VT.getHalfNumVectorElementsVT(); + assert(isPowerOf2_32(Op.getNumOperands())); + size_t HalfNumOps = Op.getNumOperands() / 2; + SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().take_front(HalfNumOps)); + SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().drop_front(HalfNumOps)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); + } + unsigned NumOpElts = Op.getOperand(0).getSimpleValueType().getVectorMinNumElements(); SDValue Vec = DAG.getUNDEF(VT); diff --git a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll index 87d95d7596d4..139579b3d2a3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll +++ b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll @@ -161,72 +161,71 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) { define <128 x i1> @fv128(ptr %p, i64 %index, i64 %tc) { ; CHECK-LABEL: fv128: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_0) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_0) +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vid.v v16 -; CHECK-NEXT: vsaddu.vx v16, v16, a1 -; CHECK-NEXT: vmsltu.vx v0, v16, a2 -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 4, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_1) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_1) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) +; CHECK-NEXT: vle8.v v9, (a0) ; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 6, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 4 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v10, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v8, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_2) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_2) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 8, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 6 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_3) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_3) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 10, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 8 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_4) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_4) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 12, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 10 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vid.v v16 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v0, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_5) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_5) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 14, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 12 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_6) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_6) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v0, v16, 14 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v0, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v0, v8, 8 ; CHECK-NEXT: ret %mask = call <128 x i1> @llvm.get.active.lane.mask.v128i1.i64(i64 %index, i64 %tc) ret <128 x i1> %mask diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll index c64216180c2a..ed434deea1a8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll +++ b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll @@ -19,7 +19,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV32-NEXT: th.swia a0, (a1), 4, 0 ; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV32-NEXT: vle8.v v10, (a3) -; RV32-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV32-NEXT: vslideup.vi v10, v9, 4 ; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV32-NEXT: vzext.vf4 v12, v10 @@ -42,7 +42,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV64-NEXT: th.swia a0, (a1), 4, 0 ; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV64-NEXT: vle8.v v10, (a3) -; RV64-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV64-NEXT: vslideup.vi v10, v9, 4 ; RV64-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV64-NEXT: vzext.vf4 v12, v10 diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll index 76aa2b913c65..e15e6452163b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll @@ -469,9 +469,8 @@ define @extract_nxv6f16_nxv12f16_6( %in) ; CHECK: # %bb.0: ; CHECK-NEXT: csrr a0, vlenb ; CHECK-NEXT: srli a0, a0, 2 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vslidedown.vx v12, v9, a0 ; CHECK-NEXT: add a1, a0, a0 ; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll index e5bef20fd9e2..98e6b8f2dd76 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll @@ -19,14 +19,11 @@ define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) { define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) { ; CHECK-LABEL: concat_4xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v14, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 6 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -37,24 +34,18 @@ define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x i32> %d, <1 x i32> %e, <1 x i32> %f, <1 x i32> %g, <1 x i32> %h) { ; CHECK-LABEL: concat_8xv1i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v18, v13 -; CHECK-NEXT: vmv1r.v v20, v11 -; CHECK-NEXT: vmv1r.v v22, v9 -; CHECK-NEXT: vsetivli zero, 2, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v22, 1 -; CHECK-NEXT: vsetivli zero, 3, e32, m2, tu, ma +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 1 +; CHECK-NEXT: vslideup.vi v12, v13, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 2 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 1 +; CHECK-NEXT: vslideup.vi v8, v9, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vslideup.vi v8, v10, 2 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 3 -; CHECK-NEXT: vsetivli zero, 5, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 4 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v18, 5 -; CHECK-NEXT: vsetivli zero, 7, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 6 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 7 +; CHECK-NEXT: vslideup.vi v8, v12, 4 ; CHECK-NEXT: ret %ab = shufflevector <1 x i32> %a, <1 x i32> %b, <2 x i32> %cd = shufflevector <1 x i32> %c, <1 x i32> %d, <2 x i32> @@ -80,15 +71,14 @@ define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) { define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) { ; CHECK-LABEL: concat_4xv4i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v16, v10 -; CHECK-NEXT: vmv1r.v v20, v9 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 4 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v16, 8 +; CHECK-NEXT: vmv1r.v v14, v11 +; CHECK-NEXT: vmv1r.v v12, v10 +; CHECK-NEXT: vmv1r.v v10, v9 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 12 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> @@ -99,26 +89,18 @@ define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d, <2 x i32> %e, <2 x i32> %f, <2 x i32> %g, <2 x i32> %h) { ; CHECK-LABEL: concat_8xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v20, v14 -; CHECK-NEXT: vmv1r.v v24, v13 -; CHECK-NEXT: vmv1r.v v28, v11 -; CHECK-NEXT: vmv1r.v v4, v10 -; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v0, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v4, 4 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v28, 6 -; CHECK-NEXT: vsetivli zero, 10, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v24, 10 -; CHECK-NEXT: vsetivli zero, 14, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 12 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 2 +; CHECK-NEXT: vslideup.vi v12, v13, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 14 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -152,29 +134,27 @@ define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) { define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) { ; VLA-LABEL: concat_4xv8i32: ; VLA: # %bb.0: -; VLA-NEXT: vmv2r.v v16, v14 -; VLA-NEXT: vmv2r.v v24, v12 -; VLA-NEXT: vmv2r.v v0, v10 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 8 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 16 +; VLA-NEXT: vmv2r.v v20, v14 +; VLA-NEXT: vmv2r.v v16, v12 +; VLA-NEXT: vmv2r.v v12, v10 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_4xv8i32: ; VLS: # %bb.0: -; VLS-NEXT: vmv2r.v v16, v14 -; VLS-NEXT: vmv2r.v v24, v12 -; VLS-NEXT: vmv2r.v v0, v10 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 8 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 16 +; VLS-NEXT: vmv2r.v v20, v14 +; VLS-NEXT: vmv2r.v v16, v12 +; VLS-NEXT: vmv2r.v v12, v10 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> %cd = shufflevector <8 x i32> %c, <8 x i32> %d, <16 x i32> @@ -185,123 +165,49 @@ define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h) { ; VLA-LABEL: concat_8xv4i32: ; VLA: # %bb.0: -; VLA-NEXT: addi sp, sp, -16 -; VLA-NEXT: .cfi_def_cfa_offset 16 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: sub sp, sp, a0 -; VLA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLA-NEXT: vmv1r.v v16, v15 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v14 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v13 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLA-NEXT: vmv1r.v v18, v15 +; VLA-NEXT: vmv1r.v v20, v14 +; VLA-NEXT: vmv1r.v v22, v13 ; VLA-NEXT: vmv1r.v v16, v12 -; VLA-NEXT: addi a0, sp, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v0, v11 -; VLA-NEXT: vmv1r.v v24, v10 -; VLA-NEXT: vmv1r.v v16, v9 -; VLA-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v16, 4 -; VLA-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 8 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 12 -; VLA-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 16 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 20 -; VLA-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vmv1r.v v14, v11 +; VLA-NEXT: vmv1r.v v12, v10 +; VLA-NEXT: vmv1r.v v10, v9 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v20, v18, 4 +; VLA-NEXT: vslideup.vi v16, v22, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v12, v14, 4 +; VLA-NEXT: vslideup.vi v8, v10, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 28 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: add sp, sp, a0 -; VLA-NEXT: addi sp, sp, 16 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_8xv4i32: ; VLS: # %bb.0: -; VLS-NEXT: addi sp, sp, -16 -; VLS-NEXT: .cfi_def_cfa_offset 16 -; VLS-NEXT: addi sp, sp, -512 -; VLS-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLS-NEXT: vmv1r.v v16, v15 -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v14 -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v13 -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLS-NEXT: vmv1r.v v18, v15 +; VLS-NEXT: vmv1r.v v20, v14 +; VLS-NEXT: vmv1r.v v22, v13 ; VLS-NEXT: vmv1r.v v16, v12 -; VLS-NEXT: addi a0, sp, 16 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v0, v11 -; VLS-NEXT: vmv1r.v v24, v10 -; VLS-NEXT: vmv1r.v v16, v9 -; VLS-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v16, 4 -; VLS-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 8 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 12 -; VLS-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 16 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 20 -; VLS-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vmv1r.v v14, v11 +; VLS-NEXT: vmv1r.v v12, v10 +; VLS-NEXT: vmv1r.v v10, v9 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v20, v18, 4 +; VLS-NEXT: vslideup.vi v16, v22, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v12, v14, 4 +; VLS-NEXT: vslideup.vi v8, v10, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 28 -; VLS-NEXT: addi sp, sp, 512 -; VLS-NEXT: addi sp, sp, 16 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll index 4ec2e59672ad..657d52354aa3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll @@ -27,13 +27,14 @@ define void @widen_3xv4i16(ptr %x, ptr %z) { ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: addi a2, a0, 8 -; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: vle16.v v9, (a2) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: vle16.v v12, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v10, (a0) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 8 +; CHECK-NEXT: vsetivli zero, 12, e16, m2, ta, ma ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -72,20 +73,18 @@ define void @widen_4xv4i16(ptr %x, ptr %z) { define void @widen_4xv4i16_unaligned(ptr %x, ptr %z) { ; CHECK-NO-MISALIGN-LABEL: widen_4xv4i16_unaligned: ; CHECK-NO-MISALIGN: # %bb.0: -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m1, ta, ma ; CHECK-NO-MISALIGN-NEXT: vle8.v v8, (a0) -; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 -; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) ; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 16 -; CHECK-NO-MISALIGN-NEXT: vle8.v v12, (a2) +; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) +; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 ; CHECK-NO-MISALIGN-NEXT: addi a0, a0, 24 -; CHECK-NO-MISALIGN-NEXT: vle8.v v14, (a0) -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NO-MISALIGN-NEXT: vle8.v v9, (a0) +; CHECK-NO-MISALIGN-NEXT: vle8.v v11, (a2) +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NO-MISALIGN-NEXT: vse16.v v8, (a1) ; CHECK-NO-MISALIGN-NEXT: ret ; @@ -187,18 +186,17 @@ define void @strided_constant_mismatch_4xv4i16(ptr %x, ptr %z) { ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) -; CHECK-NEXT: addi a2, a0, 2 -; CHECK-NEXT: vle16.v v10, (a2) ; CHECK-NEXT: addi a2, a0, 6 -; CHECK-NEXT: vle16.v v12, (a2) +; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: addi a2, a0, 2 ; CHECK-NEXT: addi a0, a0, 8 -; CHECK-NEXT: vle16.v v14, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v9, (a0) +; CHECK-NEXT: vle16.v v11, (a2) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -258,17 +256,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV32-NEXT: vle16.v v8, (a0) ; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v10, (a0) -; RV32-NEXT: add a0, a0, a4 -; RV32-NEXT: vle16.v v12, (a0) -; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v14, (a0) -; RV32-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v10, 4 -; RV32-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v12, 8 +; RV32-NEXT: add a4, a0, a4 +; RV32-NEXT: vle16.v v10, (a4) +; RV32-NEXT: add a2, a4, a2 +; RV32-NEXT: vle16.v v9, (a2) +; RV32-NEXT: vle16.v v11, (a0) +; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV32-NEXT: vslideup.vi v10, v9, 4 +; RV32-NEXT: vslideup.vi v8, v11, 4 ; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV32-NEXT: vslideup.vi v8, v14, 12 +; RV32-NEXT: vslideup.vi v8, v10, 8 ; RV32-NEXT: vse16.v v8, (a1) ; RV32-NEXT: ret ; @@ -277,17 +274,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV64-NEXT: vle16.v v8, (a0) ; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v10, (a0) -; RV64-NEXT: add a0, a0, a3 -; RV64-NEXT: vle16.v v12, (a0) -; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v14, (a0) -; RV64-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v10, 4 -; RV64-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v12, 8 +; RV64-NEXT: add a3, a0, a3 +; RV64-NEXT: vle16.v v10, (a3) +; RV64-NEXT: add a2, a3, a2 +; RV64-NEXT: vle16.v v9, (a2) +; RV64-NEXT: vle16.v v11, (a0) +; RV64-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV64-NEXT: vslideup.vi v10, v9, 4 +; RV64-NEXT: vslideup.vi v8, v11, 4 ; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV64-NEXT: vslideup.vi v8, v14, 12 +; RV64-NEXT: vslideup.vi v8, v10, 8 ; RV64-NEXT: vse16.v v8, (a1) ; RV64-NEXT: ret ; @@ -296,17 +292,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; ZVE64F-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; ZVE64F-NEXT: vle16.v v8, (a0) ; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v10, (a0) -; ZVE64F-NEXT: add a0, a0, a3 -; ZVE64F-NEXT: vle16.v v12, (a0) -; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v14, (a0) -; ZVE64F-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v10, 4 -; ZVE64F-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v12, 8 +; ZVE64F-NEXT: add a3, a0, a3 +; ZVE64F-NEXT: vle16.v v10, (a3) +; ZVE64F-NEXT: add a2, a3, a2 +; ZVE64F-NEXT: vle16.v v9, (a2) +; ZVE64F-NEXT: vle16.v v11, (a0) +; ZVE64F-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; ZVE64F-NEXT: vslideup.vi v10, v9, 4 +; ZVE64F-NEXT: vslideup.vi v8, v11, 4 ; ZVE64F-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; ZVE64F-NEXT: vslideup.vi v8, v14, 12 +; ZVE64F-NEXT: vslideup.vi v8, v10, 8 ; ZVE64F-NEXT: vse16.v v8, (a1) ; ZVE64F-NEXT: ret %a = load <4 x i16>, ptr %x diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll index eb7894ede046..b3bda5973eb8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll @@ -441,57 +441,50 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -609,57 +602,50 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -787,60 +773,53 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -1404,90 +1383,125 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1682,90 +1696,125 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 -; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 +; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1982,94 +2031,129 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -3723,57 +3807,50 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -3889,57 +3966,50 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -4066,60 +4136,53 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -4671,90 +4734,125 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -4947,90 +5045,125 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -5246,94 +5379,129 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll index f3ae03af7c78..0b236f6d3ff3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll @@ -2136,17 +2136,18 @@ define @mgather_baseidx_nxv32i8(ptr %base, ; RV64-NEXT: vluxei64.v v13, (a0), v24, v0.t ; RV64-NEXT: srli a1, a1, 2 ; RV64-NEXT: vsetvli a3, zero, e8, mf2, ta, ma -; RV64-NEXT: vslidedown.vx v0, v16, a1 -; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma -; RV64-NEXT: vsext.vf8 v16, v10 -; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu -; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t +; RV64-NEXT: vslidedown.vx v8, v16, a1 ; RV64-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; RV64-NEXT: vslidedown.vx v0, v0, a2 +; RV64-NEXT: vslidedown.vx v0, v8, a2 ; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma ; RV64-NEXT: vsext.vf8 v16, v11 ; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu ; RV64-NEXT: vluxei64.v v15, (a0), v16, v0.t +; RV64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; RV64-NEXT: vsext.vf8 v16, v10 +; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu +; RV64-NEXT: vmv1r.v v0, v8 +; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t ; RV64-NEXT: vmv4r.v v8, v12 ; RV64-NEXT: ret %ptrs = getelementptr inbounds i8, ptr %base, %idxs diff --git a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll index c27488b18a01..d13d67fd0a88 100644 --- a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll +++ b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll @@ -9,39 +9,38 @@ define <4 x float> @foo(ptr %0) nounwind { ; CHECK-NEXT: sd s0, 32(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s1, 24(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s2, 16(sp) # 8-byte Folded Spill -; CHECK-NEXT: lhu s0, 6(a0) -; CHECK-NEXT: lhu s1, 4(a0) -; CHECK-NEXT: lhu s2, 0(a0) -; CHECK-NEXT: lhu a0, 2(a0) +; CHECK-NEXT: lhu s0, 0(a0) +; CHECK-NEXT: lhu s1, 2(a0) +; CHECK-NEXT: lhu s2, 4(a0) +; CHECK-NEXT: lhu a0, 6(a0) ; CHECK-NEXT: fmv.w.x fa0, a0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 8(sp) +; CHECK-NEXT: fsw fa0, 4(sp) ; CHECK-NEXT: fmv.w.x fa0, s2 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: fsw fa0, 12(sp) ; CHECK-NEXT: fmv.w.x fa0, s1 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 12(sp) +; CHECK-NEXT: fsw fa0, 8(sp) ; CHECK-NEXT: fmv.w.x fa0, s0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 4(sp) -; CHECK-NEXT: addi a0, sp, 8 +; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: addi a0, sp, 4 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: mv a0, sp ; CHECK-NEXT: vle32.v v8, (a0) -; CHECK-NEXT: vsetivli zero, 2, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 1 ; CHECK-NEXT: addi a0, sp, 12 -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: vsetivli zero, 3, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 2 -; CHECK-NEXT: addi a0, sp, 4 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: addi a0, sp, 8 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) +; CHECK-NEXT: vle32.v v10, (a0) +; CHECK-NEXT: mv a0, sp +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 1 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: vslideup.vi v8, v9, 3 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s0, 32(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s1, 24(sp) # 8-byte Folded Reload -- GitLab From 51d5b6581912c8495360a09a0e6be978e0374d90 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:26:29 +0800 Subject: [PATCH 115/933] [RISCV] Handle scalable ops with < EEW / 2 narrow types in combineBinOp_VLToVWBinOp_VL (#84158) We can remove the restriction that the narrow type needs to be exactly EEW / 2 for scalable ISD::{ADD,SUB,MUL} nodes. This allows us to perform the combine even if we can't fully fold the extend into the widening op. VP intrinsics already do this, since they are lowered to _VL nodes which don't have this restriction. The "exactly EEW / 2" narrow type restriction prevented us from emitting V{S,Z}EXT_VL nodes with i1 element types which crash when we try to select them, since no other legal type is double the size of i1, see the test case added in this PR `i1_zext`. So to preserve this, this adds a check for i1 narrow types instead. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 8 +- .../RISCV/rvv/vscale-vw-web-simplification.ll | 38 +- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 483 ++++++++++-------- llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll | 384 +++++++------- llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll | 320 ++++++------ 5 files changed, 631 insertions(+), 602 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 71059b5bdc0f..5a2fb0239e0a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -13715,12 +13715,8 @@ struct NodeExtensionHelper { SDValue NarrowElt = OrigOperand.getOperand(0); MVT NarrowVT = NarrowElt.getSimpleValueType(); - - unsigned ScalarBits = VT.getScalarSizeInBits(); - unsigned NarrowScalarBits = NarrowVT.getScalarSizeInBits(); - - // Ensure the extension's semantic is equivalent to rvv vzext or vsext. - if (ScalarBits != NarrowScalarBits * 2) + // i1 types are legal but we can't select V{S,Z}EXT_VLs with them. + if (NarrowVT.getVectorElementType() == MVT::i1) break; SupportsZExt = Opc == ISD::ZERO_EXTEND; diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll index 972fa66917a5..e56dca0732bb 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll @@ -283,18 +283,19 @@ define @vwop_vscale_sext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_sext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vsext.vf4 v11, v8 -; FOLDING-NEXT: vsext.vf4 v8, v9 -; FOLDING-NEXT: vsext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vsext.vf2 v11, v8 +; FOLDING-NEXT: vsext.vf2 v8, v9 +; FOLDING-NEXT: vsext.vf2 v9, v10 +; FOLDING-NEXT: vwmul.vv v10, v11, v8 +; FOLDING-NEXT: vwadd.vv v8, v11, v9 +; FOLDING-NEXT: vwsub.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y @@ -563,18 +564,19 @@ define @vwop_vscale_zext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_zext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vzext.vf4 v11, v8 -; FOLDING-NEXT: vzext.vf4 v8, v9 -; FOLDING-NEXT: vzext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vzext.vf2 v11, v8 +; FOLDING-NEXT: vzext.vf2 v8, v9 +; FOLDING-NEXT: vzext.vf2 v9, v10 +; FOLDING-NEXT: vwmulu.vv v10, v11, v8 +; FOLDING-NEXT: vwaddu.vv v8, v11, v9 +; FOLDING-NEXT: vwsubu.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index a559fbf2bc8a..66a7eea18be5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s define @vwadd_vv_nxv1i64_nxv1i32( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i32: @@ -421,10 +421,10 @@ define @vwaddu_wx_nxv8i64_nxv8i32( %va, i32 define @vwadd_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -451,10 +451,10 @@ define @vwadd_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -469,10 +469,10 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -485,9 +485,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 define @vwadd_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -497,9 +497,9 @@ define @vwadd_wv_nxv1i64_nxv1i16( %va, @vwaddu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -511,9 +511,9 @@ define @vwadd_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -527,9 +527,9 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -541,10 +541,10 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 define @vwadd_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -555,10 +555,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -571,10 +571,10 @@ define @vwadd_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -589,10 +589,10 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -605,9 +605,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 define @vwadd_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -617,9 +617,9 @@ define @vwadd_wv_nxv2i64_nxv2i16( %va, @vwaddu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -631,9 +631,9 @@ define @vwadd_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -647,9 +647,9 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -661,10 +661,10 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 define @vwadd_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -675,10 +675,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -691,10 +691,10 @@ define @vwadd_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -709,10 +709,10 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -725,9 +725,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 define @vwadd_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -737,9 +737,9 @@ define @vwadd_wv_nxv4i64_nxv4i16( %va, @vwaddu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -751,9 +751,9 @@ define @vwadd_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -767,9 +767,9 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -781,10 +781,10 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 define @vwadd_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -795,10 +795,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -811,10 +811,10 @@ define @vwadd_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -829,10 +829,10 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -845,9 +845,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 define @vwadd_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -857,9 +857,9 @@ define @vwadd_wv_nxv8i64_nxv8i16( %va, @vwaddu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -871,9 +871,9 @@ define @vwadd_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -887,9 +887,9 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -901,10 +901,10 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 define @vwadd_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -915,10 +915,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -931,10 +931,10 @@ define @vwadd_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -949,10 +949,10 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -965,9 +965,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwadd_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -977,9 +977,9 @@ define @vwadd_wv_nxv1i64_nxv1i8( %va, @vwaddu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -991,9 +991,9 @@ define @vwadd_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1007,9 +1007,9 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1021,10 +1021,10 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwadd_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1035,10 +1035,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1051,10 +1051,10 @@ define @vwadd_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1069,10 +1069,10 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1085,9 +1085,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwadd_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1097,9 +1097,9 @@ define @vwadd_wv_nxv2i64_nxv2i8( %va, @vwaddu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1111,9 +1111,9 @@ define @vwadd_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1127,9 +1127,9 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1141,10 +1141,10 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwadd_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1155,10 +1155,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1171,10 +1171,10 @@ define @vwadd_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1189,10 +1189,10 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1205,9 +1205,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwadd_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1217,9 +1217,9 @@ define @vwadd_wv_nxv4i64_nxv4i8( %va, @vwaddu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1231,9 +1231,9 @@ define @vwadd_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,9 +1247,9 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1261,10 +1261,10 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwadd_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1275,10 +1275,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1291,10 +1291,10 @@ define @vwadd_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,10 +1309,10 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1325,9 +1325,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwadd_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1337,9 +1337,9 @@ define @vwadd_wv_nxv8i64_nxv8i8( %va, @vwaddu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1351,9 +1351,9 @@ define @vwadd_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1367,9 +1367,9 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1377,3 +1377,34 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % %vc = add %va, %vb ret %vc } + +; Make sure that we don't introduce any V{S,Z}EXT_VL nodes with i1 types from +; combineBinOp_VLToVWBinOp_VL, since they can't be selected. +define @i1_zext( %va, %vb, ptr %p) { +; RV32-LABEL: i1_zext: +; RV32: # %bb.0: +; RV32-NEXT: vsetvli a1, zero, e64, m1, ta, ma +; RV32-NEXT: vmv.v.i v9, 0 +; RV32-NEXT: vmerge.vim v9, v9, 1, v0 +; RV32-NEXT: vadd.vv v8, v9, v8 +; RV32-NEXT: li a1, 42 +; RV32-NEXT: sh a1, 0(a0) +; RV32-NEXT: ret +; +; RV64-LABEL: i1_zext: +; RV64: # %bb.0: +; RV64-NEXT: vsetvli a1, zero, e64, m1, ta, mu +; RV64-NEXT: vadd.vi v8, v8, 1, v0.t +; RV64-NEXT: li a1, 42 +; RV64-NEXT: sh a1, 0(a0) +; RV64-NEXT: ret + %vc = zext %va to + %vd = add %vc, %vb + +; Introduce an illegal type so that the DAG changes after legalizing +; types. Otherwise the legalize vector ops phase will be run immediately after +; the legalize types phase, and the zext will already be in non-i1 form by the +; time combineBinOp_VLToVWBinOp_VL is called. + store i9 42, ptr %p + ret %vd +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll index 3634162eefd6..539a4bdb27ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll @@ -341,10 +341,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i32( %va, i3 define @vwmul_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -355,10 +355,10 @@ define @vwmul_vv_nxv1i64_nxv1i16( %va, @vwmulu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -369,10 +369,10 @@ define @vwmulu_vv_nxv1i64_nxv1i16( %va, @vwmulsu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -385,10 +385,10 @@ define @vwmul_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -403,10 +403,10 @@ define @vwmulu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -421,10 +421,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -437,10 +437,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 define @vwmul_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwmul_vv_nxv2i64_nxv2i16( %va, @vwmulu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -465,10 +465,10 @@ define @vwmulu_vv_nxv2i64_nxv2i16( %va, @vwmulsu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -481,10 +481,10 @@ define @vwmul_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -499,10 +499,10 @@ define @vwmulu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -517,10 +517,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -533,10 +533,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 define @vwmul_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -547,10 +547,10 @@ define @vwmul_vv_nxv4i64_nxv4i16( %va, @vwmulu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -561,10 +561,10 @@ define @vwmulu_vv_nxv4i64_nxv4i16( %va, @vwmulsu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -577,10 +577,10 @@ define @vwmul_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -595,10 +595,10 @@ define @vwmulu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -613,10 +613,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -629,10 +629,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 define @vwmul_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -643,10 +643,10 @@ define @vwmul_vv_nxv8i64_nxv8i16( %va, @vwmulu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -657,10 +657,10 @@ define @vwmulu_vv_nxv8i64_nxv8i16( %va, @vwmulsu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -673,10 +673,10 @@ define @vwmul_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -691,10 +691,10 @@ define @vwmulu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -709,10 +709,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -725,10 +725,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 define @vwmul_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -739,10 +739,10 @@ define @vwmul_vv_nxv1i64_nxv1i8( %va, @vwmulu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -753,10 +753,10 @@ define @vwmulu_vv_nxv1i64_nxv1i8( %va, @vwmulsu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -769,10 +769,10 @@ define @vwmul_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -787,10 +787,10 @@ define @vwmulu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -805,10 +805,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -821,10 +821,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % define @vwmul_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -835,10 +835,10 @@ define @vwmul_vv_nxv2i64_nxv2i8( %va, @vwmulu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -849,10 +849,10 @@ define @vwmulu_vv_nxv2i64_nxv2i8( %va, @vwmulsu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -865,10 +865,10 @@ define @vwmul_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -883,10 +883,10 @@ define @vwmulu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -901,10 +901,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -917,10 +917,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % define @vwmul_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -931,10 +931,10 @@ define @vwmul_vv_nxv4i64_nxv4i8( %va, @vwmulu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -945,10 +945,10 @@ define @vwmulu_vv_nxv4i64_nxv4i8( %va, @vwmulsu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -961,10 +961,10 @@ define @vwmul_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -979,10 +979,10 @@ define @vwmulu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -997,10 +997,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1013,10 +1013,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % define @vwmul_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1027,10 +1027,10 @@ define @vwmul_vv_nxv8i64_nxv8i8( %va, @vwmulu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1041,10 +1041,10 @@ define @vwmulu_vv_nxv8i64_nxv8i8( %va, @vwmulsu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -1057,10 +1057,10 @@ define @vwmul_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1075,10 +1075,10 @@ define @vwmulu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1093,10 +1093,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll index 123469ade0ed..852814d648bf 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll @@ -421,10 +421,10 @@ define @vwsubu_wx_nxv8i64_nxv8i32( %va, i32 define @vwsub_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwsub_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -483,9 +483,9 @@ define @vwsubu_vx_nxv1i64_nxv1i16( %va, i16 define @vwsub_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -495,9 +495,9 @@ define @vwsub_wv_nxv1i64_nxv1i16( %va, @vwsubu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -509,9 +509,9 @@ define @vwsub_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -525,9 +525,9 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -539,10 +539,10 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 define @vwsub_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -569,10 +569,10 @@ define @vwsub_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -601,9 +601,9 @@ define @vwsubu_vx_nxv2i64_nxv2i16( %va, i16 define @vwsub_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -613,9 +613,9 @@ define @vwsub_wv_nxv2i64_nxv2i16( %va, @vwsubu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -627,9 +627,9 @@ define @vwsub_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -643,9 +643,9 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -657,10 +657,10 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 define @vwsub_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -687,10 +687,10 @@ define @vwsub_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -719,9 +719,9 @@ define @vwsubu_vx_nxv4i64_nxv4i16( %va, i16 define @vwsub_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -731,9 +731,9 @@ define @vwsub_wv_nxv4i64_nxv4i16( %va, @vwsubu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -745,9 +745,9 @@ define @vwsub_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -761,9 +761,9 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -775,10 +775,10 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 define @vwsub_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -805,10 +805,10 @@ define @vwsub_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -837,9 +837,9 @@ define @vwsubu_vx_nxv8i64_nxv8i16( %va, i16 define @vwsub_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -849,9 +849,9 @@ define @vwsub_wv_nxv8i64_nxv8i16( %va, @vwsubu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -863,9 +863,9 @@ define @vwsub_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -879,9 +879,9 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -893,10 +893,10 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 define @vwsub_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -923,10 +923,10 @@ define @vwsub_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -955,9 +955,9 @@ define @vwsubu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwsub_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -967,9 +967,9 @@ define @vwsub_wv_nxv1i64_nxv1i8( %va, @vwsubu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -981,9 +981,9 @@ define @vwsub_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -997,9 +997,9 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1011,10 +1011,10 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwsub_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1041,10 +1041,10 @@ define @vwsub_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1073,9 +1073,9 @@ define @vwsubu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwsub_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1085,9 +1085,9 @@ define @vwsub_wv_nxv2i64_nxv2i8( %va, @vwsubu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1099,9 +1099,9 @@ define @vwsub_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1115,9 +1115,9 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1129,10 +1129,10 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwsub_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1159,10 +1159,10 @@ define @vwsub_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1191,9 +1191,9 @@ define @vwsubu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwsub_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1203,9 +1203,9 @@ define @vwsub_wv_nxv4i64_nxv4i8( %va, @vwsubu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1217,9 +1217,9 @@ define @vwsub_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1233,9 +1233,9 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,10 +1247,10 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwsub_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1277,10 +1277,10 @@ define @vwsub_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,9 +1309,9 @@ define @vwsubu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwsub_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1321,9 +1321,9 @@ define @vwsub_wv_nxv8i64_nxv8i8( %va, @vwsubu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1335,9 +1335,9 @@ define @vwsub_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1351,9 +1351,9 @@ define @vwsubu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From d22cf4365ca58ccf1db21341d63ac49921f1c47a Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:52:19 -0400 Subject: [PATCH 116/933] [cmake] Place clang behind mlir in the liist of external projects In preparation for the initial ClangIR upstreaming process, move clang behind MLIR in the list of external projects. Otherwise, cmake will attempt to build clang before MLIR. reland of https://github.com/llvm/llvm-project/pull/86050 Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/86210 --- llvm/tools/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index c6116ac81d12..cde57367934e 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,12 +37,13 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. -add_llvm_external_project(clang) add_llvm_external_project(lld) -add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# Flang depends on mlir, so place it afterward +# ClangIR and Flang depend on mlir, lldb and Flang depend on clang, sort them +# accordingly so place them afterwards +add_llvm_external_project(clang) add_llvm_external_project(flang) +add_llvm_external_project(lldb) add_llvm_external_project(bolt) # Automatically add remaining sub-directories containing a 'CMakeLists.txt' -- GitLab From e66b670f3bf9312f696e66c31152ae535207d6bb Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:53:48 -0400 Subject: [PATCH 117/933] [CIR][Basic][NFC] Add the CIR language to the Language enum Add the CIR language to the Language enum and the standard usages of it. commit-id:fd12b2c2 Reviewers: bcardosolopes, AaronBallman, erichkeane Reviewed By: AaronBallman, bcardosolopes Pull Request: https://github.com/llvm/llvm-project/pull/86072 --- clang/include/clang/Basic/LangStandard.h | 5 +++-- clang/include/clang/Driver/Types.def | 1 + clang/lib/Basic/LangStandards.cpp | 3 +++ .../Serialization/SymbolGraphSerializer.cpp | 1 + clang/lib/Frontend/CompilerInvocation.cpp | 13 +++++++++++-- clang/lib/Frontend/FrontendActions.cpp | 1 + clang/lib/Frontend/FrontendOptions.cpp | 1 + 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h index 199e24c67316..8e25afc83366 100644 --- a/clang/include/clang/Basic/LangStandard.h +++ b/clang/include/clang/Basic/LangStandard.h @@ -26,8 +26,9 @@ enum class Language : uint8_t { /// Assembly: we accept this only so that we can preprocess it. Asm, - /// LLVM IR: we accept this so that we can run the optimizer on it, - /// and compile it to assembly or object code. + /// LLVM IR & CIR: we accept these so that we can run the optimizer on them, + /// and compile them to assembly or object code (or LLVM for CIR). + CIR, LLVM_IR, ///@{ Languages that the frontend can parse and compile. diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def index f72c27e1ee70..0e0cae5fb706 100644 --- a/clang/include/clang/Driver/Types.def +++ b/clang/include/clang/Driver/Types.def @@ -90,6 +90,7 @@ TYPE("ir", LLVM_BC, INVALID, "bc", phases TYPE("lto-ir", LTO_IR, INVALID, "s", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("lto-bc", LTO_BC, INVALID, "o", phases::Compile, phases::Backend, phases::Assemble, phases::Link) +TYPE("cir", CIR, INVALID, "cir", phases::Compile, phases::Backend, phases::Assemble, phases::Link) // Misc. TYPE("ast", AST, INVALID, "ast", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("ifs", IFS, INVALID, "ifs", phases::IfsMerge) diff --git a/clang/lib/Basic/LangStandards.cpp b/clang/lib/Basic/LangStandards.cpp index cb2c07723499..c8c9292abcb2 100644 --- a/clang/lib/Basic/LangStandards.cpp +++ b/clang/lib/Basic/LangStandards.cpp @@ -21,6 +21,8 @@ StringRef clang::languageToString(Language L) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "ClangIR"; case Language::C: return "C"; case Language::CXX: @@ -92,6 +94,7 @@ LangStandard::Kind clang::getDefaultLanguageStandard(clang::Language Lang, switch (Lang) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Invalid input kind!"); case Language::OpenCL: return LangStandard::lang_opencl12; diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 349b93e2a232..545860acb7db 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -208,6 +208,7 @@ StringRef getLanguageName(Language Lang) { case Language::Unknown: case Language::Asm: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Unsupported language kind"); } diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0df6a82ccd89..7bd91d4791ec 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2757,6 +2757,9 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, case Language::HLSL: Lang = "hlsl"; break; + case Language::CIR: + Lang = "cir"; + break; } GenerateArg(Consumer, OPT_x, @@ -2958,6 +2961,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, .Cases("ast", "pcm", "precompiled-header", InputKind(Language::Unknown, InputKind::Precompiled)) .Case("ir", Language::LLVM_IR) + .Case("cir", Language::CIR) .Default(Language::Unknown); if (DashX.isUnknown()) @@ -3323,6 +3327,7 @@ static bool IsInputCompatibleWithStandard(InputKind IK, switch (IK.getLanguage()) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("should not parse language flags for this input"); case Language::C: @@ -3388,6 +3393,8 @@ static StringRef GetInputKindName(InputKind IK) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "Clang IR"; case Language::HLSL: return "HLSL"; @@ -3403,7 +3410,8 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts, const llvm::Triple &T, InputKind IK) { if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { if (Opts.ObjCAutoRefCount) GenerateArg(Consumer, OPT_fobjc_arc); if (Opts.PICLevel != 0) @@ -3689,7 +3697,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, unsigned NumErrorsBefore = Diags.getNumErrors(); if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { // ObjCAAutoRefCount and Sanitize LangOpts are used to setup the // PassManager in BackendUtil.cpp. They need to be initialized no matter // what the input type is. diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 81fcd8d5ae9b..3fd1cdd3b479 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1083,6 +1083,7 @@ void PrintPreambleAction::ExecuteAction() { case Language::CUDA: case Language::HIP: case Language::HLSL: + case Language::CIR: break; case Language::Unknown: diff --git a/clang/lib/Frontend/FrontendOptions.cpp b/clang/lib/Frontend/FrontendOptions.cpp index bf83b27c1367..32ed99571e85 100644 --- a/clang/lib/Frontend/FrontendOptions.cpp +++ b/clang/lib/Frontend/FrontendOptions.cpp @@ -34,5 +34,6 @@ InputKind FrontendOptions::getInputKindForExtension(StringRef Extension) { .Case("hip", Language::HIP) .Cases("ll", "bc", Language::LLVM_IR) .Case("hlsl", Language::HLSL) + .Case("cir", Language::CIR) .Default(Language::Unknown); } -- GitLab From 3942bd2fb56380aa050977dc6aede011e191d9b0 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 17:05:50 -0700 Subject: [PATCH 118/933] [SLP]Fix a crash if the argument of call was affected by minbitwidt analysis. Need to support proper type conversion for function arguments to avoid compiler crash. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 9 +- .../X86/call-arg-reduced-by-minbitwidth.ll | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 36b446962c4a..7295ae0ba90b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12473,12 +12473,12 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, -1)) TysForDecl.push_back( FixedVectorType::get(CI->getType(), E->Scalars.size())); + auto *CEI = cast(VL0); for (unsigned I : seq(0, CI->arg_size())) { ValueList OpVL; // Some intrinsics have scalar arguments. This argument should not be // vectorized. if (UseIntrinsic && isVectorIntrinsicWithScalarOpAtArg(ID, I)) { - CallInst *CEI = cast(VL0); ScalarArg = CEI->getArgOperand(I); OpVecs.push_back(CEI->getArgOperand(I)); if (isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) @@ -12491,6 +12491,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); return E->VectorizedValue; } + ScalarArg = CEI->getArgOperand(I); + if (cast(OpVec->getType())->getElementType() != + ScalarArg->getType()) { + auto *CastTy = FixedVectorType::get(ScalarArg->getType(), + VecTy->getNumElements()); + OpVec = Builder.CreateIntCast(OpVec, CastTy, GetOperandSignedness(I)); + } LLVM_DEBUG(dbgs() << "SLP: OpVec[" << I << "]: " << *OpVec << "\n"); OpVecs.push_back(OpVec); if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll new file mode 100644 index 000000000000..27c9655f94d3 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll @@ -0,0 +1,82 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-pc-windows-msvc19.34.0 < %s | FileCheck %s + +define void @test(ptr %0, i8 %1, i1 %cmp12.i) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ptr [[TMP0:%.*]], i8 [[TMP1:%.*]], i1 [[CMP12_I:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i1> poison, i1 [[CMP12_I]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> poison, i8 [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: br label [[PRE:%.*]] +; CHECK: pre: +; CHECK-NEXT: [[TMP6:%.*]] = zext <8 x i8> [[TMP5]] to <8 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[TMP6]], <8 x i32> ) +; CHECK-NEXT: [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i8> +; CHECK-NEXT: [[TMP9:%.*]] = add <8 x i8> [[TMP8]], +; CHECK-NEXT: [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x i8> [[TMP9]], <8 x i8> [[TMP5]] +; CHECK-NEXT: store <8 x i8> [[TMP10]], ptr [[TMP0]], align 1 +; CHECK-NEXT: br label [[PRE]] +; +entry: + %idx11 = getelementptr i8, ptr %0, i64 1 + %idx22 = getelementptr i8, ptr %0, i64 2 + %idx33 = getelementptr i8, ptr %0, i64 3 + %idx44 = getelementptr i8, ptr %0, i64 4 + %idx55 = getelementptr i8, ptr %0, i64 5 + %idx66 = getelementptr i8, ptr %0, i64 6 + %idx77 = getelementptr i8, ptr %0, i64 7 + br label %pre + +pre: + %conv.i = zext i8 %1 to i32 + %2 = tail call i32 @llvm.umax.i32(i32 %conv.i, i32 1) + %.sroa.speculated.i = add i32 %2, 1 + %intensity.0.i = select i1 %cmp12.i, i32 %.sroa.speculated.i, i32 %conv.i + %conv14.i = trunc i32 %intensity.0.i to i8 + store i8 %conv14.i, ptr %0, align 1 + %conv.i.1 = zext i8 %1 to i32 + %3 = tail call i32 @llvm.umax.i32(i32 %conv.i.1, i32 1) + %ss1 = add i32 %3, 1 + %ii1 = select i1 %cmp12.i, i32 %ss1, i32 %conv.i.1 + %conv14.i.1 = trunc i32 %ii1 to i8 + store i8 %conv14.i.1, ptr %idx11, align 1 + %conv.i.2 = zext i8 %1 to i32 + %4 = tail call i32 @llvm.umax.i32(i32 %conv.i.2, i32 1) + %ss2 = add i32 %4, 1 + %ii2 = select i1 %cmp12.i, i32 %ss2, i32 %conv.i.2 + %conv14.i.2 = trunc i32 %ii2 to i8 + store i8 %conv14.i.2, ptr %idx22, align 1 + %conv.i.3 = zext i8 %1 to i32 + %5 = tail call i32 @llvm.umax.i32(i32 %conv.i.3, i32 1) + %ss3 = add i32 %5, 1 + %ii3 = select i1 %cmp12.i, i32 %ss3, i32 %conv.i.3 + %conv14.i.3 = trunc i32 %ii3 to i8 + store i8 %conv14.i.3, ptr %idx33, align 1 + %conv.i.4 = zext i8 %1 to i32 + %6 = tail call i32 @llvm.umax.i32(i32 %conv.i.4, i32 1) + %ss4 = add i32 %6, 1 + %ii4 = select i1 %cmp12.i, i32 %ss4, i32 %conv.i.4 + %conv14.i.4 = trunc i32 %ii4 to i8 + store i8 %conv14.i.4, ptr %idx44, align 1 + %conv.i.5 = zext i8 %1 to i32 + %7 = tail call i32 @llvm.umax.i32(i32 %conv.i.5, i32 1) + %ss5 = add i32 %7, 1 + %ii5 = select i1 %cmp12.i, i32 %ss5, i32 %conv.i.5 + %conv14.i.5 = trunc i32 %ii5 to i8 + store i8 %conv14.i.5, ptr %idx55, align 1 + %conv.i.6 = zext i8 %1 to i32 + %8 = tail call i32 @llvm.umax.i32(i32 %conv.i.6, i32 1) + %ss6 = add i32 %8, 1 + %ii6 = select i1 %cmp12.i, i32 %ss6, i32 %conv.i.6 + %conv14.i.6 = trunc i32 %ii6 to i8 + store i8 %conv14.i.6, ptr %idx66, align 1 + %conv.i.7 = zext i8 %1 to i32 + %9 = tail call i32 @llvm.umax.i32(i32 %conv.i.7, i32 1) + %ss7 = add i32 %9, 1 + %ii7 = select i1 %cmp12.i, i32 %ss7, i32 %conv.i.7 + %conv14.i.7 = trunc i32 %ii7 to i8 + store i8 %conv14.i.7, ptr %idx77, align 1 + br label %pre +} -- GitLab From 7564566779eb07e9daf41a351b09cf7607871845 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Wed, 20 Mar 2024 11:58:46 -0400 Subject: [PATCH 119/933] Reapply "Move assertion for AdjustsStack from PEI to MachineVerifier (#85698)" - The check is now actually done in both PEI and the MachineVerifier. - More .mir tests trivially updated with "adjustsStack: true" as needed. --- llvm/lib/CodeGen/MachineVerifier.cpp | 6 +++++ .../clear-dead-implicit-def-impdef.mir | 2 ++ ...plicit-def-remat-requires-impdef-check.mir | 2 ++ ...implicit-def-with-impdef-greedy-assert.mir | 2 ++ .../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 ++ .../greedy-alloc-fail-sgpr1024-spill.mir | 1 + .../ran-out-of-sgprs-allocation-failure.mir | 1 + .../CodeGen/AMDGPU/sched-crash-dbg-value.mir | 2 ++ .../AMDGPU/sgpr-spill-wrong-stack-id.mir | 1 + .../AMDGPU/snippet-copy-bundle-regression.mir | 1 + .../virtregrewrite-undef-identity-copy.mir | 1 + ...no-register-coalescing-in-returnsTwice.mir | 2 ++ .../CodeGen/Hexagon/regalloc-bad-undef.mir | 2 +- .../SystemZ/RAbasic-invalid-LR-update.mir | 2 ++ .../SystemZ/clear-liverange-spillreg.mir | 1 + llvm/test/CodeGen/SystemZ/int-cmp-56.mir | 4 +++ .../SystemZ/regcoal-subranges-update.mir | 2 ++ llvm/test/CodeGen/X86/callbr-asm-kill.mir | 1 + llvm/test/CodeGen/X86/late-remat-update.mir | 1 + llvm/test/CodeGen/X86/limit-split-cost.mir | 1 + llvm/test/CodeGen/X86/regalloc-copy-hints.mir | 1 + .../CodeGen/X86/statepoint-fastregalloc.mir | 4 +++ .../X86/statepoint-invoke-ra-enter-at-end.mir | 2 +- .../X86/statepoint-invoke-ra-hoist-copies.mir | 2 +- .../statepoint-invoke-ra-inline-spiller.mir | 2 +- ...tatepoint-invoke-ra-remove-back-copies.mir | 2 +- .../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +- .../CodeGen/X86/statepoint-vreg-folding.mir | 2 +- .../memory-operand-folding-tieddef.mir | 1 + .../InstrRef/memory-operand-load-folding.mir | 1 + .../MIR/InstrRef/phi-coalesce-subreg.mir | 1 + .../DebugInfo/MIR/InstrRef/phi-coalescing.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced2.mir | 1 + .../MIR/InstrRef/phi-regallocd-to-stack.mir | 1 + .../MIR/InstrRef/phi-through-regalloc.mir | 1 + .../MIR/InstrRef/stack-coloring-dbg-phi.mir | 1 + .../MIR/InstrRef/survives-livedebugvars.mir | 1 + .../Mips/livedebugvars-stop-trimming-loc.mir | 2 ++ .../live-debug-vars-unused-arg-debugonly.mir | 2 +- .../MIR/X86/live-debug-vars-unused-arg.mir | 2 +- .../X86/livedebugvars-crossbb-interval.mir | 1 + .../X86/live-debug-vars-intervals.mir | 4 +++ .../MachineVerifier/test_adjustsstack.mir | 26 +++++++++++++++++++ 44 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 llvm/test/MachineVerifier/test_adjustsstack.mir diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index c69d36fc7fdd..005efe48ac0c 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -3697,6 +3697,9 @@ void MachineVerifier::verifyStackFrame() { if (I.getOpcode() == FrameSetupOpcode) { if (BBState.ExitIsSetup) report("FrameSetup is after another FrameSetup", &I); + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue -= TII->getFrameTotalSize(I); BBState.ExitIsSetup = true; } @@ -3712,6 +3715,9 @@ void MachineVerifier::verifyStackFrame() { errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" << AbsSPAdj << ">.\n"; } + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue += Size; BBState.ExitIsSetup = false; } diff --git a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir index 9040937d027d..1592c86e267f 100644 --- a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir +++ b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir @@ -3,6 +3,8 @@ --- name: func tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6 diff --git a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir index aa94a03786f5..47aa34e3c011 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir @@ -22,6 +22,7 @@ name: inst_stores_to_dead_spill_implicit_def_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: @@ -59,6 +60,7 @@ body: | name: inst_stores_to_dead_spill_movimm_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir index e5395b20afd4..a5d74ef75f0a 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir @@ -4,6 +4,8 @@ --- name: widget tracksRegLiveness: true +frameInfo: + adjustsStack: true jumpTable: kind: label-difference32 entries: diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir index 3616d617f84a..5ef8a94eeaa7 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir @@ -8,6 +8,8 @@ --- name: restore_undef_copy_use tracksRegLiveness: true +frameInfo: + adjustsStack: true machineFunctionInfo: maxKernArgAlign: 1 isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir index bdd89a907790..dde84af57ed2 100644 --- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir +++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir @@ -13,6 +13,7 @@ name: greedy_fail_alloc_sgpr1024_spill tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: explicitKernArgSize: 16 diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir index 2ccc24152a9f..fdfc9b043cc9 100644 --- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir +++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir @@ -24,6 +24,7 @@ registers: - { id: 10, class: sreg_64_xexec, preferred-register: '$vcc' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir index c0d199920bd9..09037709d51d 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir @@ -181,6 +181,8 @@ legalized: false regBankSelected: false selected: false tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: - { reg: '$vgpr0', virtual-reg: '%0' } - { reg: '$vgpr1', virtual-reg: '%1' } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir index efbdbca9da6b..c6ccbd99bf89 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir @@ -78,6 +78,7 @@ name: sgpr_spill_wrong_stack_id tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir index 355829825146..f8ec6bb5d943 100644 --- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir +++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir @@ -21,6 +21,7 @@ name: kernel tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir index 3d9db687ffa1..6659e9532376 100644 --- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir @@ -20,6 +20,7 @@ name: undef_identity_copy tracksRegLiveness: true frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir index 5c59566247d8..b4bbb9be8ae4 100644 --- a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir +++ b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir @@ -86,6 +86,8 @@ --- name: main exposesReturnsTwice: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: P0, size: 80, alignment: 8, local-offset: -80 } - { id: 1, name: jb1, size: 160, alignment: 8, local-offset: -240 } diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir index 67f4dd72ea0b..9468b18bf8e4 100644 --- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir +++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir @@ -135,7 +135,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true maxCallFrameSize: 0 hasOpaqueSPAdjustment: false diff --git a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir index 3b308ce3d0d2..adeec15b1755 100644 --- a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir +++ b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -25,6 +25,8 @@ name: autogen_SD21418 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: vr128bit } - { id: 1, class: vr128bit } diff --git a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir index 7ff7d9b8b709..197c3d8551fc 100644 --- a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir +++ b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir @@ -157,6 +157,7 @@ registers: - { id: 129, class: grx32bit } - { id: 130, class: fp64bit } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir index e52fd44ae47d..3e00b6065eb9 100644 --- a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir +++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir @@ -48,6 +48,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -125,6 +126,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -202,6 +204,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -279,6 +282,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir index f709b70ff1b7..bf5855010bf9 100644 --- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir +++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir @@ -49,6 +49,8 @@ body: | --- name: segfault tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: [] body: | ; CHECK-LABEL: name: segfault diff --git a/llvm/test/CodeGen/X86/callbr-asm-kill.mir b/llvm/test/CodeGen/X86/callbr-asm-kill.mir index 86c58c4715ed..0dded37c97af 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-kill.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-kill.mir @@ -45,6 +45,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%3' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/X86/late-remat-update.mir b/llvm/test/CodeGen/X86/late-remat-update.mir index 84a78f84728c..dd4e99c6df14 100644 --- a/llvm/test/CodeGen/X86/late-remat-update.mir +++ b/llvm/test/CodeGen/X86/late-remat-update.mir @@ -66,6 +66,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/limit-split-cost.mir b/llvm/test/CodeGen/X86/limit-split-cost.mir index 6f5329e5b332..7ec0404e0f73 100644 --- a/llvm/test/CodeGen/X86/limit-split-cost.mir +++ b/llvm/test/CodeGen/X86/limit-split-cost.mir @@ -86,6 +86,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir index 13b5a541fa22..d09bcd6a6b40 100644 --- a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir +++ b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir @@ -103,6 +103,7 @@ registers: - { id: 82, class: gr32 } frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true fixedStack: - { id: 0, size: 4, alignment: 4, stack-id: default, isImmutable: true } diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 02c931067300..8bac14018a7d 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -6,6 +6,8 @@ --- name: test_relocate tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi @@ -25,6 +27,8 @@ body: | --- name: test_relocate_multi_regmasks tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir index 11968f17c70a..5f05270729fd 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir @@ -231,7 +231,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir index aae2f3870138..cf9128260f19 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir @@ -398,7 +398,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir index 87f5f0f96c50..fcebc69d9b2e 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir @@ -175,7 +175,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir index 49253968fcca..8bb39a03f7e3 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir @@ -226,7 +226,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir index 858ff3f1888b..da651039ce21 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir @@ -172,7 +172,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir index e24d5e8af1f5..d40a9a06d162 100644 --- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 8 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir index cece656d0897..5ebd1a89ae92 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir @@ -100,6 +100,7 @@ registers: - { id: 38, class: gr8 } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir index f0af93804048..b0bff30c4c82 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir @@ -87,6 +87,7 @@ liveins: - { reg: '$edi', virtual-reg: '%0' } - { reg: '$xmm0', virtual-reg: '%1' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.if.then: diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir index 51d3f7e1a6a4..d73d8a375906 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir @@ -97,6 +97,7 @@ liveins: - { reg: '$esi', virtual-reg: '%4' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir index bc1c7ebac6ce..6460263c6025 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%5' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir index d59333e73fbc..68c9bf6c89dd 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir @@ -70,6 +70,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir index ab2647d3b45a..cf17af4ba430 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir @@ -71,6 +71,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir index 0fe80980e4e6..cb35bd892eea 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir @@ -65,6 +65,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir index 2a031b295a1e..61dcec49b74c 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir @@ -94,6 +94,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir index 47a7b460e43e..e80ed2e3e8eb 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%15' } frameInfo: maxAlignment: 8 + adjustsStack: true hasCalls: true stack: - { id: 0, size: 8, alignment: 8 } diff --git a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir index 3e806e43ca9e..6dbd2cd4faa7 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir @@ -113,6 +113,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir index 35ab906efc90..5df70096e930 100644 --- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir +++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir @@ -72,6 +72,8 @@ name: fn2 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: gpr32, preferred-register: '' } - { id: 1, class: gpr32, preferred-register: '' } diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir index 3cb9da8fdfe3..4d48774a78dd 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir @@ -116,7 +116,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir index 35d12b52af89..e618f48f527b 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir index 037306a2ca95..42ee73dde965 100644 --- a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir +++ b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir @@ -100,6 +100,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir index c5b6d7381ace..3beaf8996e4f 100644 --- a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir +++ b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir @@ -99,6 +99,8 @@ --- name: f1 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, @@ -127,6 +129,8 @@ body: | --- name: f2 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, diff --git a/llvm/test/MachineVerifier/test_adjustsstack.mir b/llvm/test/MachineVerifier/test_adjustsstack.mir new file mode 100644 index 000000000000..d333737e000c --- /dev/null +++ b/llvm/test/MachineVerifier/test_adjustsstack.mir @@ -0,0 +1,26 @@ +# RUN: not --crash llc -o - -start-before=twoaddressinstruction -verify-machineinstrs %s 2>&1 \ +# RUN: | FileCheck %s +# REQUIRES: aarch64-registered-target +--- | + target triple = "aarch64-unknown-linux" + declare i32 @bar(i32) nounwind + define i32 @foo() nounwind { + call i32 @bar(i32 0) + ret i32 0 + } +... +--- +name: foo +registers: + - { id: 0, class: gpr32 } +body: | + bb.0 (%ir-block.0): + ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp + %0 = COPY $wzr + $w0 = COPY %0 + BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0 + ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp + $w0 = COPY killed %0 + RET_ReallyLR implicit $w0 +... +# CHECK-LABEL: Bad machine code: AdjustsStack not set in presence of a frame pseudo instruction. -- GitLab From 8cb2d436ca50117026a8dc901c8039d9bd39b507 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:48:22 +0800 Subject: [PATCH 120/933] [Passes] Expose parseSinglePassOption (#86117) BPF and some machine function passes need it. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++++ llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++++++----------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..c5465da8b4a5 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,8 +600,9 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -620,24 +621,28 @@ Expected parseSinglePassOption(StringRef Params, StringRef OptionName, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From a2dfc9ac7da23ccf0077081c8825a23aed1df0c0 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:49:29 +0800 Subject: [PATCH 121/933] [NewPM][AMDGPU] Add AMDGPUPassRegistry.def (#86095) Move the pass registry to a separate file, prepare for porting dag-isel. --- llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 73 +++++++++++ .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 115 +++--------------- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 4 +- .../global_atomics_iterative_scan_fp.ll | 4 +- .../global_atomics_optimizer_fp_no_rtn.ll | 4 +- 5 files changed, 96 insertions(+), 104 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def new file mode 100644 index 000000000000..90f36fadf359 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -0,0 +1,73 @@ +//===- AMDGPUPassRegistry.def - Registry of AMDGPU passes -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// AMDGPU backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass()) +MODULE_PASS("amdgpu-attributor", AMDGPUAttributorPass(*this)) +MODULE_PASS("amdgpu-lower-buffer-fat-pointers", + AMDGPULowerBufferFatPointersPass(*this)) +MODULE_PASS("amdgpu-lower-ctor-dtor", AMDGPUCtorDtorLoweringPass()) +MODULE_PASS("amdgpu-lower-module-lds", AMDGPULowerModuleLDSPass(*this)) +MODULE_PASS("amdgpu-printf-runtime-binding", AMDGPUPrintfRuntimeBindingPass()) +MODULE_PASS("amdgpu-unify-metadata", AMDGPUUnifyMetadataPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("amdgpu-codegenprepare", AMDGPUCodeGenPreparePass(*this)) +FUNCTION_PASS("amdgpu-image-intrinsic-opt", + AMDGPUImageIntrinsicOptimizerPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-arguments", + AMDGPULowerKernelArgumentsPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-attributes", + AMDGPULowerKernelAttributesPass()) +FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass()) +FUNCTION_PASS("amdgpu-promote-alloca", AMDGPUPromoteAllocaPass(*this)) +FUNCTION_PASS("amdgpu-promote-alloca-to-vector", + AMDGPUPromoteAllocaToVectorPass(*this)) +FUNCTION_PASS("amdgpu-promote-kernel-arguments", + AMDGPUPromoteKernelArgumentsPass()) +FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass()) +FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes", + AMDGPUUnifyDivergentExitNodesPass()) +FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "amdgpu-atomic-optimizer", + "AMDGPUAtomicOptimizerPass", + [=](ScanOptions Strategy) { + return AMDGPUAtomicOptimizerPass(*this, Strategy); + }, + parseAMDGPUAtomicOptimizerStrategy, "strategy=dpp|iterative|none") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index c96625092a76..f7e552177d6f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -631,107 +631,26 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { AAM.registerFunctionAnalysis(); } +static Expected +parseAMDGPUAtomicOptimizerStrategy(StringRef Params) { + if (Params.empty()) + return ScanOptions::Iterative; + Params.consume_front("strategy="); + auto Result = StringSwitch>(Params) + .Case("dpp", ScanOptions::DPP) + .Cases("iterative", "", ScanOptions::Iterative) + .Case("none", ScanOptions::None) + .Default(std::nullopt); + if (Result) + return *Result; + return make_error("invalid parameter", inconvertibleErrorCode()); +} + void AMDGPUTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [this](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-attributor") { - PM.addPass(AMDGPUAttributorPass(*this)); - return true; - } - if (PassName == "amdgpu-unify-metadata") { - PM.addPass(AMDGPUUnifyMetadataPass()); - return true; - } - if (PassName == "amdgpu-printf-runtime-binding") { - PM.addPass(AMDGPUPrintfRuntimeBindingPass()); - return true; - } - if (PassName == "amdgpu-always-inline") { - PM.addPass(AMDGPUAlwaysInlinePass()); - return true; - } - if (PassName == "amdgpu-lower-module-lds") { - PM.addPass(AMDGPULowerModuleLDSPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-buffer-fat-pointers") { - PM.addPass(AMDGPULowerBufferFatPointersPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-ctor-dtor") { - PM.addPass(AMDGPUCtorDtorLoweringPass()); - return true; - } - return false; - }); - PB.registerPipelineParsingCallback( - [this](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-simplifylib") { - PM.addPass(AMDGPUSimplifyLibCallsPass()); - return true; - } - if (PassName == "amdgpu-image-intrinsic-opt") { - PM.addPass(AMDGPUImageIntrinsicOptimizerPass(*this)); - return true; - } - if (PassName == "amdgpu-usenative") { - PM.addPass(AMDGPUUseNativeCallsPass()); - return true; - } - if (PassName == "amdgpu-promote-alloca") { - PM.addPass(AMDGPUPromoteAllocaPass(*this)); - return true; - } - if (PassName == "amdgpu-promote-alloca-to-vector") { - PM.addPass(AMDGPUPromoteAllocaToVectorPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-attributes") { - PM.addPass(AMDGPULowerKernelAttributesPass()); - return true; - } - if (PassName == "amdgpu-promote-kernel-arguments") { - PM.addPass(AMDGPUPromoteKernelArgumentsPass()); - return true; - } - if (PassName == "amdgpu-unify-divergent-exit-nodes") { - PM.addPass(AMDGPUUnifyDivergentExitNodesPass()); - return true; - } - if (PassName == "amdgpu-atomic-optimizer") { - PM.addPass( - AMDGPUAtomicOptimizerPass(*this, AMDGPUAtomicOptimizerStrategy)); - return true; - } - if (PassName == "amdgpu-codegenprepare") { - PM.addPass(AMDGPUCodeGenPreparePass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-arguments") { - PM.addPass(AMDGPULowerKernelArgumentsPass(*this)); - return true; - } - if (PassName == "amdgpu-rewrite-undef-for-phi") { - PM.addPass(AMDGPURewriteUndefForPHIPass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return AMDGPUAA(); }); - }); - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "amdgpu-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [](ModulePassManager &PM, OptimizationLevel Level) { diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index 538ef42121b8..b7a91f6fa96d 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll index fab24e10f810..86e3d9338e07 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s declare i32 @llvm.amdgcn.workitem.id.x() define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) #0 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_value( diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index cc7a45cbb6e3..e70d7347890f 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction -- GitLab From 3e4caa9da4356247444e973eb470a25adae083b0 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Fri, 22 Mar 2024 08:52:40 +0800 Subject: [PATCH 122/933] [X86] Support DomainReassignment for APX NDD instructions (#85737) --- llvm/lib/Target/X86/X86DomainReassignment.cpp | 38 ++++++++++++ .../CodeGen/X86/apx/domain-reassignment.mir | 62 +++++++++---------- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Target/X86/X86DomainReassignment.cpp b/llvm/lib/Target/X86/X86DomainReassignment.cpp index 53c0486c8697..6289b3a1df1f 100644 --- a/llvm/lib/Target/X86/X86DomainReassignment.cpp +++ b/llvm/lib/Target/X86/X86DomainReassignment.cpp @@ -650,6 +650,16 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::AND16rr, X86::KANDWrr); createReplacer(X86::XOR16rr, X86::KXORWrr); + bool HasNDD = STI->hasNDD(); + if (HasNDD) { + createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWri); + createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWri); + createReplacer(X86::NOT16r_ND, X86::KNOTWrr); + createReplacer(X86::OR16rr_ND, X86::KORWrr); + createReplacer(X86::AND16rr_ND, X86::KANDWrr); + createReplacer(X86::XOR16rr_ND, X86::KXORWrr); + } + if (STI->hasBWI()) { createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm)); createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm)); @@ -684,6 +694,23 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::XOR32rr, X86::KXORDrr); createReplacer(X86::XOR64rr, X86::KXORQrr); + if (HasNDD) { + createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDri); + createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDri); + createReplacer(X86::ADD32rr_ND, X86::KADDDrr); + createReplacer(X86::NOT32r_ND, X86::KNOTDrr); + createReplacer(X86::OR32rr_ND, X86::KORDrr); + createReplacer(X86::AND32rr_ND, X86::KANDDrr); + createReplacer(X86::XOR32rr_ND, X86::KXORDrr); + createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQri); + createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQri); + createReplacer(X86::ADD64rr_ND, X86::KADDQrr); + createReplacer(X86::NOT64r_ND, X86::KNOTQrr); + createReplacer(X86::OR64rr_ND, X86::KORQrr); + createReplacer(X86::AND64rr_ND, X86::KANDQrr); + createReplacer(X86::XOR64rr_ND, X86::KXORQrr); + } + // TODO: KTEST is not a replacement for TEST due to flag differences. Need // to prove only Z flag is used. // createReplacer(X86::TEST32rr, X86::KTESTDrr); @@ -713,6 +740,17 @@ void X86DomainReassignment::initConverters() { // createReplacer(X86::TEST16rr, X86::KTESTWrr); createReplacer(X86::XOR8rr, X86::KXORBrr); + + if (HasNDD) { + createReplacer(X86::ADD8rr_ND, X86::KADDBrr); + createReplacer(X86::ADD16rr_ND, X86::KADDWrr); + createReplacer(X86::AND8rr_ND, X86::KANDBrr); + createReplacer(X86::NOT8r_ND, X86::KNOTBrr); + createReplacer(X86::OR8rr_ND, X86::KORBrr); + createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBri); + createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBri); + createReplacer(X86::XOR8rr_ND, X86::KXORBrr); + } } #undef GET_EGPR_IF_ENABLED } diff --git a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir index dcd435619990..7352aa2b307f 100644 --- a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir +++ b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq -o - %s | FileCheck %s +# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+ndd -o - %s | FileCheck %s --- | ; ModuleID = '../test/CodeGen/X86/gpr-to-mask.ll' source_filename = "../test/CodeGen/X86/gpr-to-mask.ll" @@ -302,13 +302,13 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_8bit - %12 = SHR8ri %7, 2, implicit-def dead $eflags - %13 = SHL8ri %12, 1, implicit-def dead $eflags - %14 = NOT8r %13 - %15 = OR8rr %14, %12, implicit-def dead $eflags - %16 = AND8rr %15, %13, implicit-def dead $eflags - %17 = XOR8rr %16, %12, implicit-def dead $eflags - %18 = ADD8rr %17, %14, implicit-def dead $eflags + %12 = SHR8ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL8ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT8r_ND %13 + %15 = OR8rr_ND %14, %12, implicit-def dead $eflags + %16 = AND8rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR8rr_ND %16, %12, implicit-def dead $eflags + %18 = ADD8rr_ND %17, %14, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %18, %subreg.sub_8bit_hi @@ -421,12 +421,12 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_16bit - %12 = SHR16ri %7, 2, implicit-def dead $eflags - %13 = SHL16ri %12, 1, implicit-def dead $eflags - %14 = NOT16r %13 - %15 = OR16rr %14, %12, implicit-def dead $eflags - %16 = AND16rr %15, %13, implicit-def dead $eflags - %17 = XOR16rr %16, %12, implicit-def dead $eflags + %12 = SHR16ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL16ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT16r_ND %13 + %15 = OR16rr_ND %14, %12, implicit-def dead $eflags + %16 = AND16rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR16rr_ND %16, %12, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %17, %subreg.sub_16bit @@ -524,14 +524,14 @@ body: | %2 = COPY $zmm1 %5 = MOV32rm %0, 1, $noreg, 0, $noreg - %6 = SHR32ri %5, 2, implicit-def dead $eflags - %7 = SHL32ri %6, 1, implicit-def dead $eflags - %8 = NOT32r %7 - %9 = OR32rr %8, %6, implicit-def dead $eflags - %10 = AND32rr %9, %7, implicit-def dead $eflags - %11 = XOR32rr %10, %6, implicit-def dead $eflags + %6 = SHR32ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL32ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT32r_ND %7 + %9 = OR32rr_ND %8, %6, implicit-def dead $eflags + %10 = AND32rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR32rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN32rr %11, %9, implicit-def dead $eflags - %13 = ADD32rr %12, %11, implicit-def dead $eflags + %13 = ADD32rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -627,14 +627,14 @@ body: | %2 = COPY $zmm1 %5 = MOV64rm %0, 1, $noreg, 0, $noreg - %6 = SHR64ri %5, 2, implicit-def dead $eflags - %7 = SHL64ri %6, 1, implicit-def dead $eflags - %8 = NOT64r %7 - %9 = OR64rr %8, %6, implicit-def dead $eflags - %10 = AND64rr %9, %7, implicit-def dead $eflags - %11 = XOR64rr %10, %6, implicit-def dead $eflags + %6 = SHR64ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL64ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT64r_ND %7 + %9 = OR64rr_ND %8, %6, implicit-def dead $eflags + %10 = AND64rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR64rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN64rr %11, %9, implicit-def dead $eflags - %13 = ADD64rr %12, %11, implicit-def dead $eflags + %13 = ADD64rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU8Zrrk %2, killed %3, %1 @@ -712,7 +712,7 @@ body: | %2 = COPY $zmm1 %5 = MOVZX16rm8 %0, 1, $noreg, 0, $noreg - %6 = NOT16r %5 + %6 = NOT16r_ND %5 %3 = COPY %6 %4 = VMOVAPSZrrk %2, killed %3, %1 @@ -785,7 +785,7 @@ body: | %5 = MOVZX32rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX32rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD32rr %5, %6, implicit-def dead $eflags + %7 = ADD32rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -858,7 +858,7 @@ body: | %5 = MOVZX64rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX64rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD64rr %5, %6, implicit-def dead $eflags + %7 = ADD64rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU8Zrrk %2, killed %3, %1 -- GitLab From 8d7d581ad2a96ebe54aed0e5a626048d2e2a8d2d Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:54:11 +0800 Subject: [PATCH 123/933] Revert "[Passes] Expose parseSinglePassOption" (#86225) Reverts llvm/llvm-project#86117 --- llvm/include/llvm/Passes/PassBuilder.h | 7 ---- llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++--------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index d1232124d5d8..8817a2585646 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,13 +672,6 @@ public: return Result; } - /// Handle passes only accept one bool-valued parameter. - /// - /// \return false when Params is empty. - static Expected parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName); - private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index c5465da8b4a5..9d98ae7dde52 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,9 +600,8 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected PassBuilder::parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName) { +Expected parseSinglePassOption(StringRef Params, StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -621,28 +620,24 @@ Expected PassBuilder::parseSinglePassOption(StringRef Params, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", - "CGProfile"); + return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-mandatory", - "InlinerPass"); + return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "reuse-storage", - "CoroSplitPass"); + return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); + return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", + "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -666,21 +661,19 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "post-inline", - "EntryExitInstrumenter"); + return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); + return parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "minimal", - "LowerMatrixIntrinsics"); + return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1020,13 +1013,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1042,13 +1035,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1069,13 +1062,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 718fbbef5f18a2b7e7fc4f842b1452ae9bee581a Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Thu, 21 Mar 2024 18:14:18 -0700 Subject: [PATCH 124/933] [llvm-exegesis] Kill process that recieve a signal (#86069) Before this patch, llvm-exegesis would leave processes lingering that experienced signals like segmentation faults. They would up in a signal-delivery-stop state under the ptrace and never exit. This does not cause problems (or at least many) in llvm-exegesis as they are cleaned up after the main process exits, which usually happens quickly. However, in downstream use, when many blocks are being executed (many of which run into signals) within a single process, these processes stay around and can easily exhaust the process limit on some systems. This patch cleans them up by sending SIGKILL after information about the signal that was sent has been gathered. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 5c9848f3c688..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -342,7 +342,7 @@ private: return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (wait(NULL) == -1) { + if (waitpid(ParentOrChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); @@ -361,7 +361,7 @@ private: return SendError; int ChildStatus; - if (wait(&ChildStatus) == -1) { + if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -401,6 +401,20 @@ private: Twine(strerror(errno))); } + // Send SIGKILL rather than SIGTERM as the child process has no SIGTERM + // handlers to run, and calling SIGTERM would mean that ptrace will force + // it to block in the signal-delivery-stop for the SIGSEGV/other signals, + // and upon exit. + if (kill(ParentOrChildPID, SIGKILL) == -1) + return make_error("Failed to kill child benchmarking proces: " + + Twine(strerror(errno))); + + // Wait for the process to exit so that there are no zombie processes left + // around. + if (waitpid(ParentOrChildPID, NULL, 0) == -1) + return make_error("Failed to wait for process to die: " + + Twine(strerror(errno))); + if (ChildSignalInfo.si_signo == SIGSEGV) return make_error( reinterpret_cast(ChildSignalInfo.si_addr)); -- GitLab From 4d7f28a2c4b187f0bef3877081100786156defc7 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 10:55:58 +0800 Subject: [PATCH 125/933] [Passes] Expose parseSinglePassOption (#86226) Reland #86225, adjust the name space. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++ llvm/lib/Passes/PassBuilder.cpp | 89 ++++++++++++++------------ 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..f60f4eb3f0ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -514,6 +514,26 @@ static std::optional parseOptLevel(StringRef S) { .Default(std::nullopt); } +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { + bool Result = false; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName == OptionName) { + Result = true; + } else { + return make_error( + formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) + .str(), + inconvertibleErrorCode()); + } + } + return Result; +} + namespace { /// Parser of parameters for HardwareLoops pass. @@ -600,44 +620,29 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { - bool Result = false; - while (!Params.empty()) { - StringRef ParamName; - std::tie(ParamName, Params) = Params.split(';'); - - if (ParamName == OptionName) { - Result = true; - } else { - return make_error( - formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) - .str(), - inconvertibleErrorCode()); - } - } - return Result; -} - Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 40beb9b001a3c67c60b98fae9e999dcaa2d88717 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:14:18 -0700 Subject: [PATCH 126/933] [lldb] Handle clang::Language::CIR (#86234) commit e66b670f3bf9312f696e66c31152ae535207d6bb Author: Nathan Lanza Date: Thu Mar 21 19:53:48 2024 -0400 triggers: lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp:478:16: error: enumeration value 'CIR' not handled in switch [-Werror,-Wswitch] This patch teaches lldb to handle clang::Language::CIR the same way as clang::Language::LLVM_IR. --- lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index 3ac1cf91932c..ebcc3bc99a80 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -477,6 +477,7 @@ static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) { // Based on the base language, pick one. switch (IK.getLanguage()) { case clang::Language::Unknown: + case clang::Language::CIR: case clang::Language::LLVM_IR: case clang::Language::RenderScript: llvm_unreachable("Invalid input kind!"); -- GitLab From 4865dab04cad1c5ce47468b0a52ea968e5a5503b Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:21:11 -0700 Subject: [PATCH 127/933] [BOLT] Fix unused variable warnings This patch fixes: bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1664:20: error: unused variable 'TargetAddress' [-Werror,-Wunused-variable] bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1666:20: error: unused variable 'KeyAddress' [-Werror,-Wunused-variable] --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index b028a455a6db..303e8b18fd32 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -1679,6 +1679,8 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() { << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n'; }); + (void)TargetAddress; + (void)KeyAddress; BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(JumpAddress, -- GitLab From c67ed2f1e12e1b0e16b25606e67b67a47ca848d5 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 20:35:08 -0700 Subject: [PATCH 128/933] [SelectionDAG][RISCV] Use TypeSize version of ComputeValueVTs in TargetLowering::LowerCallTo. (#86166) This is needed to support non-intrinsic functions returning tuple types which are represented as structs with scalable vector types in IR. I suspect this may have been broken since https://reviews.llvm.org/D158115 --- .../SelectionDAG/SelectionDAGBuilder.cpp | 8 +- llvm/test/CodeGen/RISCV/rvv/calling-conv.ll | 76 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 2d63774c75e3..84df98b8a613 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -10500,14 +10500,14 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { CLI.Ins.clear(); Type *OrigRetTy = CLI.RetTy; SmallVector RetTys; - SmallVector Offsets; + SmallVector Offsets; auto &DL = CLI.DAG.getDataLayout(); - ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, 0); + ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); if (CLI.IsPostTypeLegalization) { // If we are lowering a libcall after legalization, split the return type. SmallVector OldRetTys; - SmallVector OldOffsets; + SmallVector OldOffsets; RetTys.swap(OldRetTys); Offsets.swap(OldOffsets); @@ -10519,7 +10519,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; RetTys.append(NumRegs, RegisterVT); for (unsigned j = 0; j != NumRegs; ++j) - Offsets.push_back(Offset + j * RegisterVTByteSZ); + Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ)); } } diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll index 78385a80b47e..78e8700a9fef 100644 --- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll +++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll @@ -86,3 +86,79 @@ define @caller_scalable_vector_split_indirect( @callee_scalable_vector_split_indirect( zeroinitializer, %x) ret %a } + +define {, } @caller_tuple_return() { +; RV32-LABEL: caller_tuple_return: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: call callee_tuple_return +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_return: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: call callee_tuple_return +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = call {, } @callee_tuple_return() + %b = extractvalue {, } %a, 0 + %c = extractvalue {, } %a, 1 + %d = insertvalue {, } poison, %c, 0 + %e = insertvalue {, } %d, %b, 1 + ret {, } %e +} + +declare {, } @callee_tuple_return() + +define void @caller_tuple_argument({, } %x) { +; RV32-LABEL: caller_tuple_argument: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: call callee_tuple_argument +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_argument: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: call callee_tuple_argument +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = extractvalue {, } %x, 0 + %b = extractvalue {, } %x, 1 + %c = insertvalue {, } poison, %b, 0 + %d = insertvalue {, } %c, %a, 1 + call void @callee_tuple_argument({, } %d) + ret void +} + +declare void @callee_tuple_argument({, }) -- GitLab From e1a8120a63cdb6c9567b0f68d9a0390e4f5da184 Mon Sep 17 00:00:00 2001 From: Pravin Jagtap Date: Fri, 22 Mar 2024 09:25:06 +0530 Subject: [PATCH 129/933] [AMDGPU] Support double type in atomic optimizer. (#84307) Presently the atomic optimizer supports only 32-bit operations. Plan is to extend the atomic optimizer for 64-bit operations for compute and graphics. This patch extends support for double type for `uniform values` only. Going forward, will extend the support for divergent values. Adding support for divergent values requires extending/legalizing readfirstlane, readlane, writelane, etc ops for 64-bit operations to avoid `bitcast` noise that we have currently. --------- Authored-by: Pravin Jagtap --- .../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 11 +- .../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 274 +- .../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 270 +- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 560 ++ .../global_atomics_optimizer_fp_no_rtn.ll | 420 ++ .../AMDGPU/global_atomics_scan_fadd.ll | 5578 +++++++++++++++++ .../AMDGPU/global_atomics_scan_fmax.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fmin.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fsub.ll | 5576 ++++++++++++++++ 9 files changed, 20497 insertions(+), 112 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp index 9ba74a23e8af..dbb3de76b4dd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp @@ -209,8 +209,9 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) { break; } - // Only 32-bit floating point atomic ops are supported. - if (AtomicRMWInst::isFPOperation(Op) && !I.getType()->isFloatTy()) { + // Only 32 and 64 bit floating point atomic ops are supported. + if (AtomicRMWInst::isFPOperation(Op) && + !(I.getType()->isFloatTy() || I.getType()->isDoubleTy())) { return; } @@ -920,8 +921,10 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I, Value *BroadcastI = nullptr; if (TyBitWidth == 64) { - Value *const ExtractLo = B.CreateTrunc(PHI, Int32Ty); - Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(PHI, 32), Int32Ty); + Value *CastedPhi = B.CreateBitCast(PHI, IntNTy); + Value *const ExtractLo = B.CreateTrunc(CastedPhi, Int32Ty); + Value *const ExtractHi = + B.CreateTrunc(B.CreateLShr(CastedPhi, 32), Int32Ty); CallInst *const ReadFirstLaneLo = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo); CallInst *const ReadFirstLaneHi = diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll index 255c6dedbd6e..1a76f8cf87ff 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll @@ -1090,18 +1090,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB39_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB39_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1109,20 +1120,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB39_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB39_2 +; GFX90A-NEXT: .LBB39_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB39_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB39_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1132,26 +1154,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB40_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB40_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB40_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB40_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1161,18 +1204,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB41_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB41_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1180,20 +1234,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB41_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB41_2 +; GFX90A-NEXT: .LBB41_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB41_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB41_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1203,26 +1268,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB42_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1394,37 +1480,59 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB49_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB49_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB49_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB49_2 +; GFX90A-NEXT: .LBB49_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB49_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB49_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1866,23 +1974,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB65_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB65_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB65_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB65_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1892,23 +2021,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB66_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB66_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB66_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB66_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1918,44 +2068,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB67_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v2, s0 -; GFX90A-NEXT: ds_read_b64 v[0:1], v2 +; GFX90A-NEXT: v_mov_b32_e32 v4, s0 +; GFX90A-NEXT: ds_read_b64 v[2:3], v4 ; GFX90A-NEXT: s_mov_b64 s[0:1], 0 -; GFX90A-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB67_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB67_2 +; GFX90A-NEXT: .LBB67_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB67_3 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v2, s0 -; GFX940-NEXT: ds_read_b64 v[0:1], v2 +; GFX940-NEXT: v_mov_b32_e32 v4, s0 +; GFX940-NEXT: ds_read_b64 v[2:3], v4 ; GFX940-NEXT: s_mov_b64 s[0:1], 0 -; GFX940-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX940-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB67_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB67_2 +; GFX940-NEXT: .LBB67_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll index 767d347bcfaa..a948fab8f1c1 100644 --- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll @@ -1181,18 +1181,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB42_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1200,20 +1210,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB42_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB42_2 +; GFX90A-NEXT: .LBB42_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1223,26 +1243,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB43_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB43_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB43_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB43_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1252,18 +1291,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB44_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB44_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1271,20 +1320,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB44_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB44_2 +; GFX90A-NEXT: .LBB44_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB44_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB44_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1294,26 +1353,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB45_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB45_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB45_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB45_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1485,37 +1563,57 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB52_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB52_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB52_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB52_2 +; GFX90A-NEXT: .LBB52_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB52_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB52_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -2020,23 +2118,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB70_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB70_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB70_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB70_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2046,23 +2163,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB71_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB71_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB71_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB71_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2072,46 +2208,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body -; GFX90A-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB72_3 +; GFX90A-NEXT: ; %bb.1: +; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s2 -; GFX90A-NEXT: ds_read_b64 v[0:1], v0 -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 -; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX90A-NEXT: v_mov_b32_e32 v0, s4 +; GFX90A-NEXT: ds_read_b64 v[2:3], v0 +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: v_mov_b32_e32 v4, s4 +; GFX90A-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB72_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB72_2 +; GFX90A-NEXT: .LBB72_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body -; GFX940-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB72_3 +; GFX940-NEXT: ; %bb.1: +; GFX940-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v0, s2 -; GFX940-NEXT: ds_read_b64 v[0:1], v0 -; GFX940-NEXT: v_mov_b32_e32 v2, s2 -; GFX940-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX940-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NEXT: ds_read_b64 v[2:3], v0 +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: v_mov_b32_e32 v4, s4 +; GFX940-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB72_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB72_2 +; GFX940-NEXT: .LBB72_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index b7a91f6fa96d..b71728096093 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1058,6 +1058,566 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st ret float %result } +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) +; IR-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) +; IR-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-NEXT: [[TMP27:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP28:%.*]] = fmul double [[VAL]], [[TMP27]] +; IR-NEXT: [[TMP29:%.*]] = fadd double [[TMP26]], [[TMP28]] +; IR-NEXT: br label [[TMP30]] +; IR: 30: +; IR-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) +; IR-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) +; IR-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-NEXT: [[TMP23:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0x7FF0000000000000, double [[VAL]] +; IR-NEXT: [[TMP25:%.*]] = call double @llvm.minnum.f64(double [[TMP22]], double [[TMP24]]) +; IR-NEXT: br label [[TMP26]] +; IR: 26: +; IR-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP26]] +; IR-ITERATIVE: 26: +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP27]] +; +; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-DPP-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-DPP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-DPP-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-DPP-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-DPP-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-DPP-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-DPP-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-DPP-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP26]] +; IR-DPP: 26: +; IR-DPP-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-DPP-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index e70d7347890f..b9234f47df19 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -864,6 +864,426 @@ define amdgpu_ps void @global_atomic_fadd_div_address_div_value_system_scope_str ret void } +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: br label [[TMP17]] +; IR: 17: +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: br label [[TMP13]] +; IR: 13: +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: br label [[TMP13]] +; IR-ITERATIVE: 13: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: br label [[TMP13]] +; IR-DPP: 13: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll index 96c615b974ce..4f00d48551be 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe: @@ -5408,6 +5409,5583 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll index 3cc5a4cd1d0a..622be43e7442 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll index 314c52a71d93..49d415c9eed7 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll index bc9125e326c4..7a7ddbe618b0 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe: @@ -5616,6 +5617,5581 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} -- GitLab From 4e165dd5ab7f7c022e23b645cd8f4676b03a9ec4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 21:39:18 -0700 Subject: [PATCH 130/933] [TableGen] Remove unused CodeGenHwModes argument from RegSizeInfo constructor. NFC --- llvm/utils/TableGen/InfoByHwMode.cpp | 4 ++-- llvm/utils/TableGen/InfoByHwMode.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp index 4a64421c013c..5496408bb0d3 100644 --- a/llvm/utils/TableGen/InfoByHwMode.cpp +++ b/llvm/utils/TableGen/InfoByHwMode.cpp @@ -115,7 +115,7 @@ ValueTypeByHwMode llvm::getValueTypeByHwMode(Record *Rec, return ValueTypeByHwMode(Rec, llvm::getValueType(Rec)); } -RegSizeInfo::RegSizeInfo(Record *R, const CodeGenHwModes &CGH) { +RegSizeInfo::RegSizeInfo(Record *R) { RegSize = R->getValueAsInt("RegSize"); SpillSize = R->getValueAsInt("SpillSize"); SpillAlignment = R->getValueAsInt("SpillAlignment"); @@ -139,7 +139,7 @@ void RegSizeInfo::writeToStream(raw_ostream &OS) const { RegSizeInfoByHwMode::RegSizeInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { const HwModeSelect &MS = CGH.getHwModeSelect(R); for (const HwModeSelect::PairType &P : MS.Items) { - auto I = Map.insert({P.first, RegSizeInfo(P.second, CGH)}); + auto I = Map.insert({P.first, RegSizeInfo(P.second)}); assert(I.second && "Duplicate entry?"); (void)I; } diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h index 001509e5317f..1909913c50c6 100644 --- a/llvm/utils/TableGen/InfoByHwMode.h +++ b/llvm/utils/TableGen/InfoByHwMode.h @@ -181,7 +181,7 @@ struct RegSizeInfo { unsigned SpillSize; unsigned SpillAlignment; - RegSizeInfo(Record *R, const CodeGenHwModes &CGH); + RegSizeInfo(Record *R); RegSizeInfo() = default; bool operator<(const RegSizeInfo &I) const; bool operator==(const RegSizeInfo &I) const { -- GitLab From 90454a609894ab278a87be2b9f5c49714caba8df Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Fri, 22 Mar 2024 13:23:36 +0800 Subject: [PATCH 131/933] [PowerPC][AIX] support explicit sections for -ffunction-sections (#85351) Fix crashes in https://godbolt.org/z/6voEa1o6Y --- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 6 +- .../aix-xcoff-funcsect-explicitsect.ll | 142 ++++++++++++++++++ 2 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 64cae1caa643..16942c6893a1 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -2845,9 +2845,9 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() { } void PPCAIXAsmPrinter::emitFunctionEntryLabel() { - // It's not necessary to emit the label when we have individual - // function in its own csect. - if (!TM.getFunctionSections()) + // For functions without user defined section, it's not necessary to emit the + // label when we have individual function in its own csect. + if (!TM.getFunctionSections() || MF->getFunction().hasSection()) PPCAsmPrinter::emitFunctionEntryLabel(); // Emit aliasing label for function entry point label. diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll new file mode 100644 index 000000000000..4e94228404d6 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll @@ -0,0 +1,142 @@ +; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \ +; RUN: -xcoff-traceback-table=false -filetype=obj -function-sections -o %t.o < %s +; RUN: llvm-readobj -s %t.o | FileCheck %s + +define dso_local signext i32 @foo1() section "sect" { +entry: + ret i32 1 +} + +define dso_local signext i32 @foo2() section "sect2" { +entry: + ret i32 2 +} + +define dso_local signext i32 @foo3() section "sect2" { +entry: + ret i32 3 +} + +define dso_local signext i32 @foo4() { +entry: + ret i32 4 +} + +; CHECK: Symbol {{[{][[:space:]] *}}Index: [[#INDX:]]{{[[:space:]] *}}Name: sect +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+1]] +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+2]] +; CHECK-NEXT: Name: .foo1 +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+3]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+4]] +; CHECK-NEXT: Name: sect2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+5]] +; CHECK-NEXT: SectionLen: 24 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+6]] +; CHECK-NEXT: Name: .foo2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+7]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+8]] +; CHECK-NEXT: Name: .foo3 +; CHECK-NEXT: Value (RelocatableAddress): 0x30 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+9]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+10]] +; CHECK-NEXT: Name: .foo4 +; CHECK-NEXT: Value (RelocatableAddress): 0x40 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: 16 +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } -- GitLab From 0289ae51aa375fd297f1d03d27ff517223e5e998 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 08:31:17 +0100 Subject: [PATCH 132/933] [MLIR][LLVM][SROA] Support incorrectly typed memory accesses (#85813) This commit relaxes the assumption of type consistency for LLVM dialect load and store operations in SROA. Instead, there is now a check that loads and stores are in the bounds specified by the sub-slot they access. This commit additionally removes the corresponding patterns from the type consistency pass, as they are no longer necessary. Note: It will be necessary to extend Mem2Reg with the logic for differently sized accesses as well. This is non-the-less a strict upgrade for productive flows, as the type consistency pass can produce invalid IR for some odd cases. --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 6 +- .../LLVMIR/Transforms/TypeConsistency.h | 12 -- .../mlir/Interfaces/MemorySlotInterfaces.h | 3 +- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 108 ++++++++++++-- .../LLVMIR/Transforms/TypeConsistency.cpp | 101 ------------- .../Dialect/MemRef/IR/MemRefMemorySlot.cpp | 7 +- mlir/test/Dialect/LLVMIR/sroa.mlir | 90 +++++++++++ .../test/Dialect/LLVMIR/type-consistency.mlir | 140 ++---------------- 8 files changed, 204 insertions(+), 263 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index b523374f6c06..f8f9264b3889 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -323,7 +323,8 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure, } def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_AnyPointer:$addr, @@ -402,7 +403,8 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", } def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_LoadableType:$value, diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h index b32ac56d7079..cacb241bfd7a 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h +++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h @@ -29,18 +29,6 @@ namespace LLVM { /// interpret pointee types as consistently as possible. std::unique_ptr createTypeConsistencyPass(); -/// Transforms uses of pointers to a whole struct to uses of pointers to the -/// first element of a struct. This is achieved by inserting a GEP to the first -/// element when possible. -template -class AddFieldGetterToStructDirectUse : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(User user, - PatternRewriter &rewriter) const override; -}; - /// Canonicalizes GEPs of which the base type and the pointer's type hint do not /// match. This is done by replacing the original GEP into a GEP with the type /// hint as a base type when an element of the hinted type aligns with the diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h index 56e5e96aecd1..aaa261be6553 100644 --- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h +++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h @@ -26,8 +26,7 @@ struct MemorySlot { /// Memory slot attached with information about its destructuring procedure. struct DestructurableMemorySlot : public MemorySlot { - /// Maps an index within the memory slot to the type of the pointer that - /// will be generated to access the element directly. + /// Maps an index within the memory slot to the corresponding subelement type. DenseMap elementPtrs; }; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 00b4559658fd..0ef1d105aca6 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -13,10 +13,8 @@ #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/IRMapping.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" -#include "mlir/IR/ValueRange.h" #include "mlir/Interfaces/DataLayoutInterfaces.h" #include "mlir/Interfaces/MemorySlotInterfaces.h" #include "llvm/ADT/STLExtras.h" @@ -71,12 +69,8 @@ SmallVector LLVM::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap allocaTypeMap; - for (Attribute index : llvm::make_first_range(destructuredType.value())) - allocaTypeMap.insert({index, LLVM::LLVMPointerType::get(getContext())}); - - return { - DestructurableMemorySlot{{getResult(), getElemType()}, {allocaTypeMap}}}; + return {DestructurableMemorySlot{{getResult(), getElemType()}, + *destructuredType}}; } DenseMap @@ -182,17 +176,107 @@ DeletionKind LLVM::StoreOp::removeBlockingUses( return DeletionKind::Delete; } +/// Checks if `slot` can be accessed through the provided access type. +static bool isValidAccessType(const MemorySlot &slot, Type accessType, + const DataLayout &dataLayout) { + return dataLayout.getTypeSize(accessType) <= + dataLayout.getTypeSize(slot.elemType); +} + LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { - return success(getAddr() != slot.ptr || getType() == slot.elemType); + return success(getAddr() != slot.ptr || + isValidAccessType(slot, getType(), dataLayout)); } LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { return success(getAddr() != slot.ptr || - getValue().getType() == slot.elemType); + isValidAccessType(slot, getValue().getType(), dataLayout)); +} + +/// Returns the subslot's type at the requested index. +static Type getTypeAtIndex(const DestructurableMemorySlot &slot, + Attribute index) { + auto subelementIndexMap = + slot.elemType.cast().getSubelementIndexMap(); + if (!subelementIndexMap) + return {}; + assert(!subelementIndexMap->empty()); + + // Note: Returns a null-type when no entry was found. + return subelementIndexMap->lookup(index); +} + +bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A load always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getType()) > dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; +} + +bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A store always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getValue().getType()) > + dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; } //===----------------------------------------------------------------------===// @@ -390,10 +474,8 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, auto firstLevelIndex = dyn_cast(getIndices()[1]); if (!firstLevelIndex) return false; - assert(slot.elementPtrs.contains(firstLevelIndex)); - if (!llvm::isa(slot.elementPtrs.at(firstLevelIndex))) - return false; mustBeSafelyUsed.emplace_back({getResult(), reachedType}); + assert(slot.elementPtrs.contains(firstLevelIndex)); usedIndices.insert(firstLevelIndex); return true; } diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp index b25c831bc717..3d700fe94e3b 100644 --- a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp +++ b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp @@ -49,104 +49,6 @@ static bool areBitcastCompatible(DataLayout &layout, Type lhs, Type rhs) { layout.getTypeSize(lhs) == layout.getTypeSize(rhs)); } -//===----------------------------------------------------------------------===// -// AddFieldGetterToStructDirectUse -//===----------------------------------------------------------------------===// - -/// Gets the type of the first subelement of `type` if `type` is destructurable, -/// nullptr otherwise. -static Type getFirstSubelementType(Type type) { - auto destructurable = dyn_cast(type); - if (!destructurable) - return nullptr; - - Type subelementType = destructurable.getTypeAtIndex( - IntegerAttr::get(IntegerType::get(type.getContext(), 32), 0)); - if (subelementType) - return subelementType; - - return nullptr; -} - -/// Extracts a pointer to the first field of an `elemType` from the address -/// pointer of the provided MemOp, and rewires the MemOp so it uses that pointer -/// instead. -template -static void insertFieldIndirection(MemOp op, PatternRewriter &rewriter, - Type elemType) { - PatternRewriter::InsertionGuard guard(rewriter); - - rewriter.setInsertionPointAfterValue(op.getAddr()); - SmallVector firstTypeIndices{0, 0}; - - Value properPtr = rewriter.create( - op->getLoc(), LLVM::LLVMPointerType::get(op.getContext()), elemType, - op.getAddr(), firstTypeIndices); - - rewriter.modifyOpInPlace(op, - [&]() { op.getAddrMutable().assign(properPtr); }); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - LoadOp load, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(load.getAddr(), load.getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - DataLayout layout = DataLayout::closest(load); - if (!areBitcastCompatible(layout, firstType, load.getResult().getType())) - return failure(); - - insertFieldIndirection(load, rewriter, inconsistentElementType); - - // If the load does not use the first type but a type that can be casted from - // it, add a bitcast and change the load type. - if (firstType != load.getResult().getType()) { - rewriter.setInsertionPointAfterValue(load.getResult()); - BitcastOp bitcast = rewriter.create( - load->getLoc(), load.getResult().getType(), load.getResult()); - rewriter.modifyOpInPlace(load, - [&]() { load.getResult().setType(firstType); }); - rewriter.replaceAllUsesExcept(load.getResult(), bitcast.getResult(), - bitcast); - } - - return success(); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - StoreOp store, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(store.getAddr(), store.getValue().getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - - DataLayout layout = DataLayout::closest(store); - // Check that the first field has the right type or can at least be bitcast - // to the right type. - if (!areBitcastCompatible(layout, firstType, store.getValue().getType())) - return failure(); - - insertFieldIndirection(store, rewriter, inconsistentElementType); - - rewriter.modifyOpInPlace( - store, [&]() { store.getValueMutable().assign(store.getValue()); }); - - return success(); -} - //===----------------------------------------------------------------------===// // CanonicalizeAlignedGep //===----------------------------------------------------------------------===// @@ -684,9 +586,6 @@ struct LLVMTypeConsistencyPass : public LLVM::impl::LLVMTypeConsistencyBase { void runOnOperation() override { RewritePatternSet rewritePatterns(&getContext()); - rewritePatterns.add>(&getContext()); - rewritePatterns.add>( - &getContext()); rewritePatterns.add(&getContext()); rewritePatterns.add(&getContext(), maxVectorSplitSize); rewritePatterns.add(&getContext()); diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp index 7be4056fb2fc..6c5250d527ad 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp @@ -120,11 +120,8 @@ memref::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap indexMap; - for (auto const &[index, type] : *destructuredType) - indexMap.insert({index, MemRefType::get({}, type)}); - - return {DestructurableMemorySlot{{getMemref(), memrefType}, indexMap}}; + return { + DestructurableMemorySlot{{getMemref(), memrefType}, *destructuredType}}; } DenseMap diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index 02d25f27f978..ca49b1298b0e 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -215,3 +215,93 @@ llvm.func @no_nested_dynamic_indexing(%arg: i32) -> i32 { // CHECK: llvm.return %[[RES]] : i32 llvm.return %3 : i32 } + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field +llvm.func @store_first_field(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %{{.*}}, %[[ALLOCA]] : i32 + llvm.store %arg, %1 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field_different_type +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_first_field_different_type(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_sub_field +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_sub_field(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field +llvm.func @load_first_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> i32 + %2 = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %2 : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field_different_type +llvm.func @load_first_field_different_type() -> f32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> f32 + %2 = llvm.load %1 : !llvm.ptr -> f32 + // CHECK: llvm.return %[[RES]] : f32 + llvm.return %2 : f32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_sub_field +llvm.func @load_sub_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 : (i32) -> !llvm.ptr + %1 = llvm.alloca %0 x !llvm.struct<(i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] + %res = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %res : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @vector_store_type_mismatch +// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> +llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x vector<4xf32> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr + llvm.return +} diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 021151b929d8..a6176142f174 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -26,63 +26,6 @@ llvm.func @same_address_keep_inbounds(%arg: i32) { // ----- -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field -llvm.func @struct_store_instead_of_first_field(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] : i32 - llvm.store %arg, %1 : i32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field_same_size -// CHECK-SAME: (%[[ARG:.*]]: f32) -llvm.func @struct_store_instead_of_first_field_same_size(%arg: f32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK-DAG: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK-DAG: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK-DAG: %[[BITCAST:.*]] = llvm.bitcast %[[ARG]] : f32 to i32 - // CHECK: llvm.store %[[BITCAST]], %[[GEP]] : i32 - llvm.store %arg, %1 : f32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field -llvm.func @struct_load_instead_of_first_field() -> i32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[RES:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - %2 = llvm.load %1 : !llvm.ptr -> i32 - // CHECK: llvm.return %[[RES]] : i32 - llvm.return %2 : i32 -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field_same_size -llvm.func @struct_load_instead_of_first_field_same_size() -> f32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[LOADED:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - // CHECK: %[[RES:.*]] = llvm.bitcast %[[LOADED]] : i32 to f32 - %2 = llvm.load %1 : !llvm.ptr -> f32 - // CHECK: llvm.return %[[RES]] : f32 - llvm.return %2 : f32 -} - -// ----- - // CHECK-LABEL: llvm.func @index_in_final_padding llvm.func @index_in_final_padding(%arg: i32) { %0 = llvm.mlir.constant(1 : i32) : i32 @@ -135,22 +78,6 @@ llvm.func @index_not_in_padding_because_packed(%arg: i16) { // ----- -// CHECK-LABEL: llvm.func @index_to_struct -// CHECK-SAME: (%[[ARG:.*]]: i32) -llvm.func @index_to_struct(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"bar", (i32, i32)> - %7 = llvm.getelementptr %1[4] : (!llvm.ptr) -> !llvm.ptr, i8 - // CHECK: llvm.store %[[ARG]], %[[GEP1]] - llvm.store %arg, %7 : i32, !llvm.ptr - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index llvm.func @no_crash_on_negative_gep_index() { %0 = llvm.mlir.constant(1.000000e+00 : f16) : f16 @@ -175,10 +102,9 @@ llvm.func @coalesced_store_ints(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> @@ -225,11 +151,9 @@ llvm.func @coalesced_store_floats(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (f32, f32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (f32, f32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i32 to f32 - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> @@ -298,10 +222,9 @@ llvm.func @coalesced_store_packed_struct(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", packed (i16, i32, i16)> %1 = llvm.alloca %0 x !llvm.struct<"foo", packed (i16, i32, i16)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> @@ -328,9 +251,8 @@ llvm.func @vector_write_split(%arg: vector<4xi32>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] : vector<4xi32> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] : i32, !llvm.ptr + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] : i32, !llvm.ptr // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] : vector<4xi32> // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> @@ -405,36 +327,6 @@ llvm.func @vector_write_split_struct(%arg: vector<2xi64>) { // ----- -// CHECK-LABEL: llvm.func @type_consistent_vector_store -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xi32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xi32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xi32>)> - // CHECK: llvm.store %[[ARG]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @type_consistent_vector_store_other_type -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store_other_type(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xf32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xf32>)> - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[ARG]] : vector<4xi32> to vector<4xf32> - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - // CHECK-NOT: llvm.store %[[ARG]], %[[ALLOCA]] - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @bitcast_insertion // CHECK-SAME: %[[ARG:.*]]: i32 llvm.func @bitcast_insertion(%arg: i32) { @@ -478,10 +370,9 @@ llvm.func @coalesced_store_ints_subaggregate(%arg: i64) { %3 = llvm.getelementptr %1[0, 1, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> // CHECK: %[[TOP_GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[TOP_GEP]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> @@ -520,10 +411,9 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -531,8 +421,7 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> @@ -557,14 +446,12 @@ llvm.func @overlapping_vector_aggregate_store(%arg: vector<4 x i16>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP1]] + // CHECK: llvm.store %[[EXTRACT]], %[[GEP0]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST2]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> @@ -593,10 +480,9 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -604,8 +490,7 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> @@ -651,10 +536,9 @@ llvm.func @coalesced_store_ints_array(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.array<2 x i32> %1 = llvm.alloca %0 x !llvm.array<2 x i32> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> -- GitLab From c5f839bd58e7f888acc4cb39a18e9e5bbaa9fb0a Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Fri, 22 Mar 2024 09:50:42 +0100 Subject: [PATCH 133/933] [reland][libc] Add reverse_iterator comparisons (#86188) This is a reland of #86147 but with a proper `base()` function. https://en.cppreference.com/w/cpp/iterator/reverse_iterator/operator_cmp --- libc/src/__support/CPP/iterator.h | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index c5bfb1912c7b..b0fd5c9f22ae 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,6 +20,7 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; + using value_type = T; }; template class reverse_iterator { @@ -27,6 +28,8 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; + using value_type = typename iterator_traits::value_type; + using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -38,6 +41,38 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} + LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() == rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() != rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() > rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() >= rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() < rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() <= rhs.base(); + } + + LIBC_INLINE constexpr iterator_type base() const { return current; } + LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 5f1b2cffe5fab0aa733fc8d5f1546c1c800faac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 09:37:43 +0000 Subject: [PATCH 134/933] [mlir][vector] Add support for masks in castAwayContractionLeadingOneDim (#81906) Updates `castAwayContractionLeadingOneDim` to inherit from `MaskableOpRewritePattern` so that this pattern can support masking. Builds on top of #83827 --- .../Vector/Transforms/VectorTransforms.h | 6 +- .../mlir/Dialect/Vector/Utils/VectorUtils.h | 10 +- .../Transforms/VectorDropLeadUnitDim.cpp | 50 +++++---- .../vector-dropleadunitdim-transforms.mlir | 104 +++++++++++++----- 4 files changed, 114 insertions(+), 56 deletions(-) diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h index 08d3bb157a0e..1f7d6411cd5a 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h @@ -110,8 +110,10 @@ void transferOpflowOpt(RewriterBase &rewriter, Operation *rootOp); /// Cast away the leading unit dim, if exists, for the given contract op. /// Return success if the transformation applies; return failure otherwise. -LogicalResult castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, - RewriterBase &rewriter); +FailureOr +castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + RewriterBase &rewriter); } // namespace vector } // namespace mlir diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h index 35e76a8b623a..2c548fb67402 100644 --- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h +++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h @@ -127,8 +127,8 @@ SmallVector getMixedSizesXfer(bool hasTensorSemantics, /// responsible for providing an updated ("rewritten") version of: /// a. the source Op when mask _is not_ present, /// b. the source Op and the masking Op when mask _is_ present. -/// Note that the return value from `matchAndRewriteMaskableOp` depends on the -/// case above. +/// To use this pattern, implement `matchAndRewriteMaskableOp`. Note that +/// the return value will depend on the case above. template struct MaskableOpRewritePattern : OpRewritePattern { using OpRewritePattern::OpRewritePattern; @@ -162,9 +162,9 @@ private: } public: - // Matches SourceOp that can potentially be masked with `maskingOp`. If the - // latter is present, returns an updated masking op (with a replacement for - // `sourceOp` nested inside). Otherwise, returns an updated `sourceOp`. + // Matches `sourceOp` that can potentially be masked with `maskingOp`. If the + // latter is present, returns a replacement for `maskingOp`. Otherwise, + // returns a replacement for `sourceOp`. virtual FailureOr matchAndRewriteMaskableOp(SourceOp sourceOp, MaskingOpInterface maskingOp, PatternRewriter &rewriter) const = 0; diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp index 74382b027c2f..593c1e53557a 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp @@ -329,12 +329,10 @@ struct CastAwayTransferWriteLeadingOneDim } // namespace -LogicalResult +FailureOr mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, RewriterBase &rewriter) { - // TODO(#78787): Not supported masked op yet. - if (cast(contractOp.getOperation()).isMasked()) - return failure(); VectorType oldAccType = dyn_cast(contractOp.getAccType()); if (oldAccType == nullptr) return failure(); @@ -368,6 +366,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, SmallVector operands = {contractOp.getLhs(), contractOp.getRhs(), contractOp.getAcc()}; SmallVector newOperands; + auto loc = contractOp.getLoc(); for (const auto &it : llvm::enumerate(oldIndexingMaps)) { // Check if the dim to be dropped exists as a leading dim in the operand @@ -405,7 +404,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, map = AffineMap::get(map.getNumDims(), 0, transposeResults, contractOp.getContext()); operands[it.index()] = rewriter.create( - contractOp.getLoc(), operands[it.index()], perm); + loc, operands[it.index()], perm); } } // We have taken care to have the dim to be dropped be @@ -429,18 +428,29 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, // Extract if its a valid extraction, otherwise use the operand // without extraction. newOperands.push_back( - validExtract ? rewriter.create(contractOp.getLoc(), - operands[it.index()], - splatZero(dropDim)) + validExtract ? rewriter.create( + loc, operands[it.index()], splatZero(dropDim)) : operands[it.index()]); } - auto newContractOp = rewriter.create( - contractOp.getLoc(), newOperands[0], newOperands[1], newOperands[2], + + // Depending on whether this vector.contract is masked, the replacing Op + // should either be a new vector.contract Op or vector.mask Op. + Operation *newOp = rewriter.create( + loc, newOperands[0], newOperands[1], newOperands[2], rewriter.getAffineMapArrayAttr(newIndexingMaps), rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind()); - rewriter.replaceOpWithNewOp( - contractOp, contractOp->getResultTypes()[0], newContractOp); - return success(); + + if (maskingOp) { + auto newMask = rewriter.create(loc, maskingOp.getMask(), + splatZero(dropDim)); + + newOp = mlir::vector::maskOperation(rewriter, newOp, newMask); + } + + return rewriter + .create(loc, contractOp->getResultTypes()[0], + newOp->getResults()[0]) + .getResult(); } namespace { @@ -450,12 +460,14 @@ namespace { /// 1 dimensions. Also performs tranpose of lhs and rhs operands if required /// prior to extract. struct CastAwayContractionLeadingOneDim - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(vector::ContractionOp contractOp, - PatternRewriter &rewriter) const override { - return castAwayContractionLeadingOneDim(contractOp, rewriter); + : public MaskableOpRewritePattern { + using MaskableOpRewritePattern::MaskableOpRewritePattern; + + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + PatternRewriter &rewriter) const override { + return castAwayContractionLeadingOneDim(contractOp, maskingOp, rewriter); } }; diff --git a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir index af6e636245b0..4ba51c5953d1 100644 --- a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir @@ -30,6 +30,80 @@ func.func @cast_away_contraction_leading_one_dims(%arg0: vector<1x16x8xf32>, %ar } // ----- +// CHECK: #[[$MAP_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK: #[[$MAP_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK: #[[$MAP_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_const_mask +// CHECK: %[[MASK:.*]] = vector.constant_mask [15, 15, 8] : vector<16x16x8xi1> +// CHECK: %[[R0:.*]] = vector.extract %{{.*}}[0] : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}}[0] : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}}[0] : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[MASK]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP_0]], #[[$MAP_1]], #[[$MAP_2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_const_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { + %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0 : vector<1x16x16xf32> +} + +// ----- +// CHECK-DAG: #[[$MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_mask +// CHECK: %[[R0:.*]] = vector.extract %{{.*}} : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}} : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}} : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[M:.*]] = vector.extract %{{.*}} : vector<16x16x8xi1> from vector<1x16x16x8xi1> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[M]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK-NEXT: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_mask( + %arg0: vector<1x16x8xf32>, + %arg1: vector<1x8x16xf32>, + %arg2: vector<1x16x16xf32>, + %mask: vector<1x16x16x8xi1>) -> vector<1x16x16xf32> { + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0: vector<1x16x16xf32> +} + +// ----- + // CHECK-DAG: #[[$map0:.*]] = affine_map<(d0, d1) -> (d1)> // CHECK-DAG: #[[$map1:.*]] = affine_map<(d0, d1) -> (d1, d0)> // CHECK-DAG: #[[$map2:.*]] = affine_map<(d0, d1) -> (d0)> @@ -164,36 +238,6 @@ func.func @cast_away_contraction_leading_one_dims_nonleadingunitdim_rank4_acctra return %0: vector<1x1x2x16xf32> } -// ----- - -// CHECK-DAG: #[[MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)> -// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)> -// CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)> - -// CHECK-LABEL: not_insert_cast_for_contraction_under_mask -// CHECK: %[[MASK:.+]] = vector.constant_mask -// CHECK: %[[CASTED_MASK:.+]] = vector.broadcast %[[MASK]] -// CHECK: %[[RET:.+]] = vector.mask %[[CASTED_MASK]] { -// CHECK-SAME: vector.contract {{.*}} : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> } -// CHECK: return %[[RET]] : vector<1x16x16xf32> - -#contraction_accesses0 = [ - affine_map<(l, i, j, k) -> (l, i, k)>, - affine_map<(l, i, j, k) -> (l, k, j)>, - affine_map<(l, i, j, k) -> (l, i, j)> -] -#contraction_trait0 = { - indexing_maps = #contraction_accesses0, - iterator_types = ["parallel", "parallel", "parallel", "reduction"] -} - -func.func @not_insert_cast_for_contraction_under_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { - %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> - %0 = vector.mask %mask { - vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> - } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> - return %0 : vector<1x16x16xf32> -} // ----- // CHECK-LABEL: func @cast_away_extract_strided_slice_leading_one_dims -- GitLab From 99d8c25b3104fc07f46532bd681515c5f3c71133 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 09:55:18 +0000 Subject: [PATCH 135/933] [AArch64] Extra tests for v2i8 concat loads. NFC --- llvm/test/CodeGen/AArch64/insert-subvector.ll | 129 +++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/insert-subvector.ll b/llvm/test/CodeGen/AArch64/insert-subvector.ll index d7656e1cd341..95ad9807ed63 100644 --- a/llvm/test/CodeGen/AArch64/insert-subvector.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector.ll @@ -374,6 +374,131 @@ define <16 x i8> @load_v16i8_8_2(float %tmp, <16 x i8> %b, ptr %a) { ret <16 x i8> %s2 } +define <8 x i8> @load_v8i8_2_1(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[0], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_15(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_15: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: mov v0.b[1], v0.b[4] +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: tbl v0.8b, { v0.16b }, v1.8b +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_2(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[1], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_3(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_3: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[2], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_4(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_4: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[3], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_1(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: mov v0.s[1], v1.s[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_2(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.s[1], v2.s[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + ; i16 define <8 x i16> @load_v8i16_2_1(float %tmp, <8 x i16> %b, ptr %a) { @@ -400,10 +525,10 @@ define <8 x i16> @load_v8i16_2_15(float %tmp, <8 x i16> %b, ptr %a) { ; CHECK-NEXT: add x9, x0, #2 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $q0_q1 ; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: adrp x8, .LCPI40_0 ; CHECK-NEXT: ld1 { v2.h }[2], [x9] ; CHECK-NEXT: xtn v0.4h, v2.4s -; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI40_0] ; CHECK-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b ; CHECK-NEXT: ret %l = load <2 x i16>, ptr %a -- GitLab From 465ea0bfa69aa48afef58666b084467a1c96c81b Mon Sep 17 00:00:00 2001 From: Crefeda Rodrigues <65665931+cfRod@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:08:03 +0000 Subject: [PATCH 136/933] [mlir][vector] Propagate scalability in TransferWriteNonPermutationLowering (#85632) Updates `extendVectorRank` so that scalability in patterns that use it (in particular, `TransferWriteNonPermutationLowering`), is correctly propagated. Closed related previous PR https://github.com/llvm/llvm-project/pull/85270 --------- Signed-off-by: Crefeda Rodrigues Co-authored-by: Benjamin Maxwell --- .../Vector/Transforms/LowerVectorTransfer.cpp | 8 ++++++-- .../vector-transfer-permutation-lowering.mlir | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp index 4a5e8fcfb6ed..0693aa596cb2 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp @@ -41,8 +41,12 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec, SmallVector newShape(addedRank, 1); newShape.append(originalVecType.getShape().begin(), originalVecType.getShape().end()); - VectorType newVecType = - VectorType::get(newShape, originalVecType.getElementType()); + + SmallVector newScalableDims(addedRank, false); + newScalableDims.append(originalVecType.getScalableDims().begin(), + originalVecType.getScalableDims().end()); + VectorType newVecType = VectorType::get( + newShape, originalVecType.getElementType(), newScalableDims); return builder.create(loc, newVecType, vec); } diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir index 13e07f59a72a..31bd19c0be8e 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir @@ -41,6 +41,24 @@ func.func @permutation_with_mask_scalable(%2: memref, %dim_1: index, %d return %1 : vector<8x[4]x2xf32> } +// CHECK: func.func @permutation_with_mask_transfer_write_scalable( +// CHECK-SAME: %[[ARG_0:.*]]: vector<4x[8]xi16>, +// CHECK-SAME: %[[ARG_1:.*]]: memref<1x4x?x1x1x1x1xi16>, +// CHECK-SAME: %[[MASK:.*]]: vector<4x[8]xi1>) { +// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK: %[[BCAST_1:.*]] = vector.broadcast %[[ARG_0]] : vector<4x[8]xi16> to vector<1x1x1x1x4x[8]xi16> +// CHECK: %[[BCAST_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x1x1x1x4x[8]xi1> +// CHECK: %[[TRANSPOSE_1:.*]] = vector.transpose %[[BCAST_2]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi1> to vector<4x[8]x1x1x1x1xi1> +// CHECK: %[[TRANSPOSE_2:.*]] = vector.transpose %[[BCAST_1]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi16> to vector<4x[8]x1x1x1x1xi16> +// CHECK: vector.transfer_write %[[TRANSPOSE_2]], %[[ARG_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[TRANSPOSE_1]] {in_bounds = [true, true, true, true, true, true]} : vector<4x[8]x1x1x1x1xi16>, memref<1x4x?x1x1x1x1xi16> +// CHECK: return +func.func @permutation_with_mask_transfer_write_scalable(%arg0: vector<4x[8]xi16>, %arg1: memref<1x4x?x1x1x1x1xi16>, %mask: vector<4x[8]xi1>){ + %c0 = arith.constant 0 : index + vector.transfer_write %arg0, %arg1[%c0, %c0, %c0, %c0, %c0, %c0, %c0], %mask {in_bounds = [true, true], permutation_map = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d1, d2)> +} : vector<4x[8]xi16>, memref<1x4x?x1x1x1x1xi16> + + return +} module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { %f = transform.structured.match ops{["func.func"]} in %module_op -- GitLab From de7a50fb88faa1dafee33f10149561936214062b Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Fri, 22 Mar 2024 11:13:04 +0100 Subject: [PATCH 137/933] [flang] Fix lowering of host associated cray pointee symbols (#86121) Cray pointee symbols can be host associated from a module or host procedure while the related cray pointer is not explicitly associated. This caused the "not yet implemented: lowering symbol to HLFIR" to fire when lowering a reference to the cray pointee and fetching the cray pointer. This patch: - Ensures cray pointers are always instantiated when instantiating a cray pointee. - Fix internal procedure lowering to deal with cray pointee host association like it does for pointers (the lowering strategy for cray pointee is to create a pointer that is updated with the cray pointer value before being fetched). This should fix the bug reported in https://github.com/llvm/llvm-project/issues/85420. --- flang/include/flang/Lower/ConvertVariable.h | 6 +- flang/include/flang/Semantics/tools.h | 3 + flang/lib/Lower/Bridge.cpp | 5 +- flang/lib/Lower/ConvertExpr.cpp | 10 +- flang/lib/Lower/ConvertExprToHLFIR.cpp | 9 +- flang/lib/Lower/ConvertVariable.cpp | 48 ++++----- flang/lib/Lower/HostAssociations.cpp | 9 +- flang/lib/Lower/PFTBuilder.cpp | 9 ++ flang/lib/Semantics/tools.cpp | 12 +++ flang/test/Lower/HLFIR/cray-pointers.f90 | 114 ++++++++++++++++++-- flang/test/Lower/cray-pointer.f90 | 4 +- 11 files changed, 176 insertions(+), 53 deletions(-) diff --git a/flang/include/flang/Lower/ConvertVariable.h b/flang/include/flang/Lower/ConvertVariable.h index ab30e317d1d9..d70d3268acac 100644 --- a/flang/include/flang/Lower/ConvertVariable.h +++ b/flang/include/flang/Lower/ConvertVariable.h @@ -161,9 +161,9 @@ void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, fir::FortranVariableFlagsEnum::None, bool force = false); -/// For the given Cray pointee symbol return the corresponding -/// Cray pointer symbol. Assert if the pointer symbol cannot be found. -Fortran::semantics::SymbolRef getCrayPointer(Fortran::semantics::SymbolRef sym); +/// Given the Fortran type of a Cray pointee, return the fir.box type used to +/// track the cray pointee as Fortran pointer. +mlir::Type getCrayPointeeBoxType(mlir::Type); } // namespace lower } // namespace Fortran diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h index dc3cd6c894a2..66774b51316c 100644 --- a/flang/include/flang/Semantics/tools.h +++ b/flang/include/flang/Semantics/tools.h @@ -282,6 +282,9 @@ const Symbol *FindExternallyVisibleObject( // specific procedure of the same name, return it instead. const Symbol &BypassGeneric(const Symbol &); +// Given a cray pointee symbol, returns the related cray pointer symbol. +const Symbol &GetCrayPointer(const Symbol &crayPointee); + using SomeExpr = evaluate::Expr; bool ExprHasTypeCategory( diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index c3cb9ba6a47e..0b54ee818e3c 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -3995,11 +3995,12 @@ private: sym->Rank() == 0) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(*sym); + const Fortran::semantics::Symbol &ptrSym = + Fortran::semantics::GetCrayPointer(*sym); fir::ExtendedValue ptr = getSymbolExtendedValue(ptrSym, nullptr); mlir::Value ptrVal = fir::getBase(ptr); - mlir::Type ptrTy = genType(*ptrSym); + mlir::Type ptrTy = genType(ptrSym); fir::ExtendedValue pte = getSymbolExtendedValue(*sym, nullptr); diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp index d157db2cde49..fb7807718ff8 100644 --- a/flang/lib/Lower/ConvertExpr.cpp +++ b/flang/lib/Lower/ConvertExpr.cpp @@ -862,7 +862,8 @@ public: addr); } else if (sym->test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(sym); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(sym)}; ExtValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = converter.genType(*ptrSym); @@ -1537,8 +1538,8 @@ public: auto baseSym = getFirstSym(aref); if (baseSym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(baseSym); - + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(baseSym)}; fir::ExtendedValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = ptrVal.getType(); @@ -6946,7 +6947,8 @@ private: ComponentPath &components) { mlir::Value ptrVal = nullptr; if (x.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { - auto ptrSym = Fortran::lower::getCrayPointer(x); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(x)}; ExtValue ptr = converter.getSymbolExtendedValue(ptrSym); ptrVal = fir::getBase(ptr); } diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp index c5bfbdf6b8c1..fe5ce4b17b25 100644 --- a/flang/lib/Lower/ConvertExprToHLFIR.cpp +++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp @@ -284,7 +284,7 @@ private: // value of the Cray pointer variable. fir::FirOpBuilder &builder = getBuilder(); fir::FortranVariableOpInterface ptrVar = - gen(Fortran::lower::getCrayPointer(symbolRef)); + gen(Fortran::semantics::GetCrayPointer(symbolRef)); mlir::Value ptrAddr = ptrVar.getBase(); // Reinterpret the reference to a Cray pointer so that @@ -306,9 +306,16 @@ private: } return *varDef; } + llvm::errs() << *symbolRef << "\n"; TODO(getLoc(), "lowering symbol to HLFIR"); } + fir::FortranVariableOpInterface + gen(const Fortran::semantics::Symbol &symbol) { + Fortran::evaluate::SymbolRef symref{symbol}; + return gen(symref); + } + fir::FortranVariableOpInterface gen(const Fortran::evaluate::Component &component) { if (Fortran::semantics::IsAllocatableOrPointer(component.GetLastSymbol())) diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp index 94d849862099..e07ae42dc749 100644 --- a/flang/lib/Lower/ConvertVariable.cpp +++ b/flang/lib/Lower/ConvertVariable.cpp @@ -1554,6 +1554,11 @@ fir::FortranVariableFlagsAttr Fortran::lower::translateSymbolAttributes( mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym, fir::FortranVariableFlagsEnum extraFlags) { fir::FortranVariableFlagsEnum flags = extraFlags; + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + // CrayPointee are represented as pointers. + flags = flags | fir::FortranVariableFlagsEnum::pointer; + return fir::FortranVariableFlagsAttr::get(mlirContext, flags); + } const auto &attrs = sym.attrs(); if (attrs.test(Fortran::semantics::Attr::ALLOCATABLE)) flags = flags | fir::FortranVariableFlagsEnum::allocatable; @@ -1615,8 +1620,6 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, (!Fortran::semantics::IsProcedure(sym) || Fortran::semantics::IsPointer(sym)) && !sym.detailsIf()) { - bool isCrayPointee = - sym.test(Fortran::semantics::Symbol::Flag::CrayPointee); fir::FirOpBuilder &builder = converter.getFirOpBuilder(); const mlir::Location loc = genLocation(converter, sym); mlir::Value shapeOrShift; @@ -1636,31 +1639,21 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), sym); - if (isCrayPointee) { - mlir::Type baseType = - hlfir::getFortranElementOrSequenceType(base.getType()); - if (auto seqType = mlir::dyn_cast(baseType)) { - // The pointer box's sequence type must be with unknown shape. - llvm::SmallVector shape(seqType.getDimension(), - fir::SequenceType::getUnknownExtent()); - baseType = fir::SequenceType::get(shape, seqType.getEleTy()); - } - fir::BoxType ptrBoxType = - fir::BoxType::get(fir::PointerType::get(baseType)); + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + mlir::Type ptrBoxType = + Fortran::lower::getCrayPointeeBoxType(base.getType()); mlir::Value boxAlloc = builder.createTemporary(loc, ptrBoxType); // Declare a local pointer variable. - attributes = fir::FortranVariableFlagsAttr::get( - builder.getContext(), fir::FortranVariableFlagsEnum::pointer); auto newBase = builder.create( loc, boxAlloc, name, /*shape=*/nullptr, lenParams, attributes); - mlir::Value nullAddr = - builder.createNullConstant(loc, ptrBoxType.getEleTy()); + mlir::Value nullAddr = builder.createNullConstant( + loc, llvm::cast(ptrBoxType).getEleTy()); // If the element type is known-length character, then // EmboxOp does not need the length parameters. if (auto charType = mlir::dyn_cast( - fir::unwrapSequenceType(baseType))) + hlfir::getFortranElementType(base.getType()))) if (!charType.hasDynamicLen()) lenParams.clear(); @@ -2346,16 +2339,13 @@ void Fortran::lower::createRuntimeTypeInfoGlobal( defineGlobal(converter, var, globalName, linkage); } -Fortran::semantics::SymbolRef -Fortran::lower::getCrayPointer(Fortran::semantics::SymbolRef sym) { - assert(!sym->GetUltimate().owner().crayPointers().empty() && - "empty Cray pointer/pointee map"); - for (const auto &[pointee, pointer] : - sym->GetUltimate().owner().crayPointers()) { - if (pointee == sym->name()) { - Fortran::semantics::SymbolRef v{pointer.get()}; - return v; - } +mlir::Type Fortran::lower::getCrayPointeeBoxType(mlir::Type fortranType) { + mlir::Type baseType = hlfir::getFortranElementOrSequenceType(fortranType); + if (auto seqType = mlir::dyn_cast(baseType)) { + // The pointer box's sequence type must be with unknown shape. + llvm::SmallVector shape(seqType.getDimension(), + fir::SequenceType::getUnknownExtent()); + baseType = fir::SequenceType::get(shape, seqType.getEleTy()); } - llvm_unreachable("corresponding Cray pointer cannot be found"); + return fir::BoxType::get(fir::PointerType::get(baseType)); } diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp index 414673b00f44..8eb548eb2bd5 100644 --- a/flang/lib/Lower/HostAssociations.cpp +++ b/flang/lib/Lower/HostAssociations.cpp @@ -315,7 +315,11 @@ class CapturedAllocatableAndPointer public: static mlir::Type getType(Fortran::lower::AbstractConverter &converter, const Fortran::semantics::Symbol &sym) { - return fir::ReferenceType::get(converter.genType(sym)); + mlir::Type baseType = converter.genType(sym); + if (sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) + return fir::ReferenceType::get( + Fortran::lower::getCrayPointeeBoxType(baseType)); + return fir::ReferenceType::get(baseType); } static void instantiateHostTuple(const InstantiateHostTuple &args, Fortran::lower::AbstractConverter &converter, @@ -507,7 +511,8 @@ walkCaptureCategories(T visitor, Fortran::lower::AbstractConverter &converter, if (Fortran::semantics::IsProcedure(sym)) return CapturedProcedure::visit(visitor, converter, sym, ba); ba.analyze(sym); - if (Fortran::semantics::IsAllocatableOrPointer(sym)) + if (Fortran::semantics::IsAllocatableOrPointer(sym) || + sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) return CapturedAllocatableAndPointer::visit(visitor, converter, sym, ba); if (ba.isArray()) return CapturedArrays::visit(visitor, converter, sym, ba); diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp index 1dacd5cf64cd..f196b9c5a0cb 100644 --- a/flang/lib/Lower/PFTBuilder.cpp +++ b/flang/lib/Lower/PFTBuilder.cpp @@ -1594,6 +1594,11 @@ private: if (!s->has()) depth = std::max(analyze(s) + 1, depth); } + + // Make sure cray pointer is instantiated even if it is not visible. + if (ultimate.test(Fortran::semantics::Symbol::Flag::CrayPointee)) + depth = std::max( + analyze(Fortran::semantics::GetCrayPointer(ultimate)) + 1, depth); adjustSize(depth + 1); bool global = lower::symbolIsGlobal(sym); layeredVarList[depth].emplace_back(sym, global, depth); @@ -2002,6 +2007,10 @@ struct SymbolVisitor { } } } + // - CrayPointer needs to be available whenever a CrayPointee is used. + if (symbol.GetUltimate().test( + Fortran::semantics::Symbol::Flag::CrayPointee)) + visitSymbol(Fortran::semantics::GetCrayPointer(symbol)); } template diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp index 0484baae93cd..2230047abd72 100644 --- a/flang/lib/Semantics/tools.cpp +++ b/flang/lib/Semantics/tools.cpp @@ -403,6 +403,18 @@ const Symbol &BypassGeneric(const Symbol &symbol) { return symbol; } +const Symbol &GetCrayPointer(const Symbol &crayPointee) { + const Symbol *found{nullptr}; + for (const auto &[pointee, pointer] : + crayPointee.GetUltimate().owner().crayPointers()) { + if (pointee == crayPointee.name()) { + found = &pointer.get(); + break; + } + } + return DEREF(found); +} + bool ExprHasTypeCategory( const SomeExpr &expr, const common::TypeCategory &type) { auto dynamicType{expr.GetType()}; diff --git a/flang/test/Lower/HLFIR/cray-pointers.f90 b/flang/test/Lower/HLFIR/cray-pointers.f90 index d1f1a5647ff1..d969aa5d747a 100644 --- a/flang/test/Lower/HLFIR/cray-pointers.f90 +++ b/flang/test/Lower/HLFIR/cray-pointers.f90 @@ -204,14 +204,14 @@ end subroutine test7 ! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.array<5xi32> {bindc_name = "arr", uniq_name = "_QFtest7Earr"} ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_3]](%[[VAL_4]]) {uniq_name = "_QFtest7Earr"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} +! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_6:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest7Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> -! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} -! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref> ! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -226,14 +226,14 @@ subroutine test8() end subroutine test8 ! CHECK-LABEL: func.func @_QPtest8( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest8Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -256,14 +256,14 @@ subroutine test9() end subroutine test9 ! CHECK-LABEL: func.func @_QPtest9( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest9Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -287,12 +287,12 @@ subroutine test10() end subroutine test10 ! CHECK-LABEL: func.func @_QPtest10( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest10Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -315,12 +315,12 @@ subroutine test11() end subroutine test11 ! CHECK-LABEL: func.func @_QPtest11( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest11Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -330,3 +330,97 @@ end subroutine test11 ! CHECK: %[[VAL_14:.*]] = fir.box_addr %[[VAL_13]] : (!fir.box>) -> !fir.ptr ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ptr ! CHECK: fir.call @_QPsub2(%[[VAL_15]]) fastmath : (i32) -> () + +module test_mod + integer(8) :: cray_pointer + real :: cray_pointee + pointer(cray_pointer, cray_pointee) +end module + +subroutine test_hidden_pointer + ! Only the pointee is accessed, yet the pointer is needed + ! for lowering. + use test_mod, only : cray_pointee + call takes_real(cray_pointee) +end +! CHECK-LABEL: func.func @_QPtest_hidden_pointer() { +! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QMtest_modEcray_pointer) : !fir.ref +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMtest_modEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QMtest_modEcray_pointee"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr +! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> +! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_6]] : !fir.ref> +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_7]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_10:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_8]], %[[VAL_9]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_12:.*]] = fir.box_addr %[[VAL_11]] : (!fir.box>) -> !fir.ptr +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.ref +! CHECK: fir.call @_QPtakes_real(%[[VAL_13]]) fastmath : (!fir.ref) -> () +! CHECK: return +! CHECK: } + + + +subroutine test_craypointer_capture(n) + integer :: n + character(n) :: cray_pointee + integer(8) :: cray_pointer + pointer(cray_pointer, cray_pointee) + call internal() + contains +subroutine internal() + call takes_character(cray_pointee) +end subroutine +end subroutine +! CHECK-LABEL: func.func @_QPtest_craypointer_capture( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref {fir.bindc_name = "n"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_craypointer_captureEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "cray_pointer", uniq_name = "_QFtest_craypointer_captureEcray_pointer"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_7:.*]] = arith.cmpi sgt, %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_8:.*]] = arith.select %[[VAL_7]], %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_8]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, i32) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> +! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]] typeparams %[[VAL_8]] : (!fir.ptr>, i32) -> !fir.box>> +! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> +! CHECK: %[[VAL_12:.*]] = fir.alloca tuple>>>, !fir.ref> +! CHECK: %[[VAL_13:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_13]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: fir.store %[[VAL_9]]#1 to %[[VAL_14]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_15]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: fir.store %[[VAL_4]]#1 to %[[VAL_16]] : !fir.llvm_ptr> +! CHECK: fir.call @_QFtest_craypointer_capturePinternal(%[[VAL_12]]) fastmath : (!fir.ref>>>, !fir.ref>>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func private @_QFtest_craypointer_capturePinternal( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>>, !fir.ref>> {fir.host_assoc}) +! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref>>> +! CHECK: %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box>>) -> index +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]] typeparams %[[VAL_5]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, index) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_7]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.llvm_ptr> +! CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_10]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref> +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref>>>) -> !fir.ref> +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_15:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_13]], %[[VAL_14]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref>>> +! CHECK: %[[VAL_17:.*]] = fir.box_addr %[[VAL_16]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[VAL_18:.*]] = fir.emboxchar %[[VAL_17]], %[[VAL_5]] : (!fir.ptr>, index) -> !fir.boxchar<1> +! CHECK: fir.call @_QPtakes_character(%[[VAL_18]]) fastmath : (!fir.boxchar<1>) -> () +! CHECK: return +! CHECK: } diff --git a/flang/test/Lower/cray-pointer.f90 b/flang/test/Lower/cray-pointer.f90 index 4e9f49daab4e..06910bce35a1 100644 --- a/flang/test/Lower/cray-pointer.f90 +++ b/flang/test/Lower/cray-pointer.f90 @@ -264,8 +264,8 @@ subroutine cray_array() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<3xi32> {{.*}} -! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[c2:.*]] = arith.constant 2 : i64 ! CHECK: %[[c1:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c2]], %[[c1]] : i64 @@ -327,8 +327,8 @@ subroutine cray_arraySection() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c2:.*]] = arith.constant 2 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<2xi32> {{.*}} -! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[c1:.*]] = arith.constant 2 : i64 ! CHECK: %[[c0:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c1]], %[[c0]] : i64 -- GitLab From 66f88de80599ec4461b0fdac3d1e396b6e83052d Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 138/933] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 312 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 377 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..005cdebc0d8a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..904f0c371f44 --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,312 @@ +// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cb4453dc69d75064c9a82e9a6a9bf0d0ded4b204 Mon Sep 17 00:00:00 2001 From: XChy Date: Fri, 22 Mar 2024 18:35:20 +0800 Subject: [PATCH 139/933] [SelectionDAG] Prevent combination on inconsistent type in `combineCarryDiamond` (#84888) Fixes #84831 When matching carry pattern with `getAsCarry`, it may produce different type of carryout. This patch checks such case and does early exit. I'm new to DAG, any suggestion is appreciated. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 +++++- llvm/test/CodeGen/X86/addcarry.ll | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 7009f375df11..db81f9199170 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3473,6 +3473,11 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, return SDValue(); if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) return SDValue(); + // Guarantee identical type of CarryOut + EVT CarryOutType = N->getValueType(0); + if (CarryOutType != Carry0.getValue(1).getValueType() || + CarryOutType != Carry1.getValue(1).getValueType()) + return SDValue(); // Canonicalize the add/sub of A and B (the top node in the above ASCII art) // as Carry0 and the add/sub of the carry in as Carry1 (the middle node). @@ -3520,7 +3525,7 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, // TODO: match other operations that can merge flags (ADD, etc) DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0)); if (N->getOpcode() == ISD::AND) - return DAG.getConstant(0, DL, MVT::i1); + return DAG.getConstant(0, DL, CarryOutType); return Merged.getValue(1); } diff --git a/llvm/test/CodeGen/X86/addcarry.ll b/llvm/test/CodeGen/X86/addcarry.ll index 3fc4ed99fad0..f8d32fc2d292 100644 --- a/llvm/test/CodeGen/X86/addcarry.ll +++ b/llvm/test/CodeGen/X86/addcarry.ll @@ -1490,3 +1490,26 @@ define { i64, i64 } @addcarry_commutative_2(i64 %x0, i64 %x1, i64 %y0, i64 %y1) %r1 = insertvalue { i64, i64 } %r0, i64 %b1s, 1 ret { i64, i64 } %r1 } + +define i1 @pr84831(i64 %arg) { +; CHECK-LABEL: pr84831: +; CHECK: # %bb.0: +; CHECK-NEXT: testq %rdi, %rdi +; CHECK-NEXT: setne %al +; CHECK-NEXT: xorl %ecx, %ecx +; CHECK-NEXT: addb $-1, %al +; CHECK-NEXT: adcq $1, %rcx +; CHECK-NEXT: setb %al +; CHECK-NEXT: retq + %a = icmp ult i64 0, %arg + %add1 = add i64 0, 1 + %carryout1 = icmp ult i64 %add1, 0 + %b = zext i1 %a to i64 + %add2 = add i64 %add1, %b + %carryout2 = icmp ult i64 %add2, %add1 + %zc1 = zext i1 %carryout1 to i63 + %zc2 = zext i1 %carryout2 to i63 + %or = or i63 %zc1, %zc2 + %trunc = trunc i63 %or to i1 + ret i1 %trunc +} -- GitLab From 6e755c51a916dc521ffe89738bcab47a5442ad06 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:48:35 +0800 Subject: [PATCH 140/933] Revert "[RISCV] Support RISC-V Profiles in -march option (#76357)" This reverts commit 66f88de80599ec4461b0fdac3d1e396b6e83052d as there are some failures. --- clang/docs/ReleaseNotes.rst | 1 - clang/test/Driver/riscv-profiles.c | 312 ----------------------------- llvm/lib/Support/RISCVISAInfo.cpp | 64 ------ 3 files changed, 377 deletions(-) delete mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 005cdebc0d8a..fd12bb41be47 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,7 +520,6 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. -- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c deleted file mode 100644 index 904f0c371f44..000000000000 --- a/clang/test/Driver/riscv-profiles.c +++ /dev/null @@ -1,312 +0,0 @@ -// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s -// RVI20U32: "-target-feature" "-a" -// RVI20U32: "-target-feature" "-c" -// RVI20U32: "-target-feature" "-d" -// RVI20U32: "-target-feature" "-f" -// RVI20U32: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s -// RVI20U64: "-target-feature" "-a" -// RVI20U64: "-target-feature" "-c" -// RVI20U64: "-target-feature" "-d" -// RVI20U64: "-target-feature" "-f" -// RVI20U64: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s -// RVA20U64: "-target-feature" "+m" -// RVA20U64: "-target-feature" "+a" -// RVA20U64: "-target-feature" "+f" -// RVA20U64: "-target-feature" "+d" -// RVA20U64: "-target-feature" "+c" -// RVA20U64: "-target-feature" "+ziccamoa" -// RVA20U64: "-target-feature" "+ziccif" -// RVA20U64: "-target-feature" "+zicclsm" -// RVA20U64: "-target-feature" "+ziccrse" -// RVA20U64: "-target-feature" "+zicntr" -// RVA20U64: "-target-feature" "+zicsr" -// RVA20U64: "-target-feature" "+za128rs" - -// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s -// RVA20S64: "-target-feature" "+m" -// RVA20S64: "-target-feature" "+a" -// RVA20S64: "-target-feature" "+f" -// RVA20S64: "-target-feature" "+d" -// RVA20S64: "-target-feature" "+c" -// RVA20S64: "-target-feature" "+ziccamoa" -// RVA20S64: "-target-feature" "+ziccif" -// RVA20S64: "-target-feature" "+zicclsm" -// RVA20S64: "-target-feature" "+ziccrse" -// RVA20S64: "-target-feature" "+zicntr" -// RVA20S64: "-target-feature" "+zicsr" -// RVA20S64: "-target-feature" "+zifencei" -// RVA20S64: "-target-feature" "+za128rs" -// RVA20S64: "-target-feature" "+ssccptr" -// RVA20S64: "-target-feature" "+sstvala" -// RVA20S64: "-target-feature" "+sstvecd" -// RVA20S64: "-target-feature" "+svade" -// RVA20S64: "-target-feature" "+svbare" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s -// RVA22U64: "-target-feature" "+m" -// RVA22U64: "-target-feature" "+a" -// RVA22U64: "-target-feature" "+f" -// RVA22U64: "-target-feature" "+d" -// RVA22U64: "-target-feature" "+c" -// RVA22U64: "-target-feature" "+zic64b" -// RVA22U64: "-target-feature" "+zicbom" -// RVA22U64: "-target-feature" "+zicbop" -// RVA22U64: "-target-feature" "+zicboz" -// RVA22U64: "-target-feature" "+ziccamoa" -// RVA22U64: "-target-feature" "+ziccif" -// RVA22U64: "-target-feature" "+zicclsm" -// RVA22U64: "-target-feature" "+ziccrse" -// RVA22U64: "-target-feature" "+zicntr" -// RVA22U64: "-target-feature" "+zicsr" -// RVA22U64: "-target-feature" "+zihintpause" -// RVA22U64: "-target-feature" "+zihpm" -// RVA22U64: "-target-feature" "+za64rs" -// RVA22U64: "-target-feature" "+zfhmin" -// RVA22U64: "-target-feature" "+zba" -// RVA22U64: "-target-feature" "+zbb" -// RVA22U64: "-target-feature" "+zbs" -// RVA22U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s -// RVA22S64: "-target-feature" "+m" -// RVA22S64: "-target-feature" "+a" -// RVA22S64: "-target-feature" "+f" -// RVA22S64: "-target-feature" "+d" -// RVA22S64: "-target-feature" "+c" -// RVA22S64: "-target-feature" "+zic64b" -// RVA22S64: "-target-feature" "+zicbom" -// RVA22S64: "-target-feature" "+zicbop" -// RVA22S64: "-target-feature" "+zicboz" -// RVA22S64: "-target-feature" "+ziccamoa" -// RVA22S64: "-target-feature" "+ziccif" -// RVA22S64: "-target-feature" "+zicclsm" -// RVA22S64: "-target-feature" "+ziccrse" -// RVA22S64: "-target-feature" "+zicntr" -// RVA22S64: "-target-feature" "+zicsr" -// RVA22S64: "-target-feature" "+zifencei" -// RVA22S64: "-target-feature" "+zihintpause" -// RVA22S64: "-target-feature" "+zihpm" -// RVA22S64: "-target-feature" "+za64rs" -// RVA22S64: "-target-feature" "+zfhmin" -// RVA22S64: "-target-feature" "+zba" -// RVA22S64: "-target-feature" "+zbb" -// RVA22S64: "-target-feature" "+zbs" -// RVA22S64: "-target-feature" "+zkt" -// RVA22S64: "-target-feature" "+ssccptr" -// RVA22S64: "-target-feature" "+sscounterenw" -// RVA22S64: "-target-feature" "+sstvala" -// RVA22S64: "-target-feature" "+sstvecd" -// RVA22S64: "-target-feature" "+svade" -// RVA22S64: "-target-feature" "+svbare" -// RVA22S64: "-target-feature" "+svinval" -// RVA22S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s -// RVA23U64: "-target-feature" "+m" -// RVA23U64: "-target-feature" "+a" -// RVA23U64: "-target-feature" "+f" -// RVA23U64: "-target-feature" "+d" -// RVA23U64: "-target-feature" "+c" -// RVA23U64: "-target-feature" "+v" -// RVA23U64: "-target-feature" "+zic64b" -// RVA23U64: "-target-feature" "+zicbom" -// RVA23U64: "-target-feature" "+zicbop" -// RVA23U64: "-target-feature" "+zicboz" -// RVA23U64: "-target-feature" "+ziccamoa" -// RVA23U64: "-target-feature" "+ziccif" -// RVA23U64: "-target-feature" "+zicclsm" -// RVA23U64: "-target-feature" "+ziccrse" -// RVA23U64: "-target-feature" "+zicntr" -// RVA23U64: "-target-feature" "+zicond" -// RVA23U64: "-target-feature" "+zicsr" -// RVA23U64: "-target-feature" "+zihintntl" -// RVA23U64: "-target-feature" "+zihintpause" -// RVA23U64: "-target-feature" "+zihpm" -// RVA23U64: "-target-feature" "+experimental-zimop" -// RVA23U64: "-target-feature" "+za64rs" -// RVA23U64: "-target-feature" "+zawrs" -// RVA23U64: "-target-feature" "+zfa" -// RVA23U64: "-target-feature" "+zfhmin" -// RVA23U64: "-target-feature" "+zcb" -// RVA23U64: "-target-feature" "+experimental-zcmop" -// RVA23U64: "-target-feature" "+zba" -// RVA23U64: "-target-feature" "+zbb" -// RVA23U64: "-target-feature" "+zbs" -// RVA23U64: "-target-feature" "+zkt" -// RVA23U64: "-target-feature" "+zvbb" -// RVA23U64: "-target-feature" "+zvfhmin" -// RVA23U64: "-target-feature" "+zvkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s -// RVA23S64: "-target-feature" "+m" -// RVA23S64: "-target-feature" "+a" -// RVA23S64: "-target-feature" "+f" -// RVA23S64: "-target-feature" "+d" -// RVA23S64: "-target-feature" "+c" -// RVA23S64: "-target-feature" "+v" -// RVA23S64: "-target-feature" "+h" -// RVA23S64: "-target-feature" "+zic64b" -// RVA23S64: "-target-feature" "+zicbom" -// RVA23S64: "-target-feature" "+zicbop" -// RVA23S64: "-target-feature" "+zicboz" -// RVA23S64: "-target-feature" "+ziccamoa" -// RVA23S64: "-target-feature" "+ziccif" -// RVA23S64: "-target-feature" "+zicclsm" -// RVA23S64: "-target-feature" "+ziccrse" -// RVA23S64: "-target-feature" "+zicntr" -// RVA23S64: "-target-feature" "+zicond" -// RVA23S64: "-target-feature" "+zicsr" -// RVA23S64: "-target-feature" "+zifencei" -// RVA23S64: "-target-feature" "+zihintntl" -// RVA23S64: "-target-feature" "+zihintpause" -// RVA23S64: "-target-feature" "+zihpm" -// RVA23S64: "-target-feature" "+experimental-zimop" -// RVA23S64: "-target-feature" "+za64rs" -// RVA23S64: "-target-feature" "+zawrs" -// RVA23S64: "-target-feature" "+zfa" -// RVA23S64: "-target-feature" "+zfhmin" -// RVA23S64: "-target-feature" "+zcb" -// RVA23S64: "-target-feature" "+experimental-zcmop" -// RVA23S64: "-target-feature" "+zba" -// RVA23S64: "-target-feature" "+zbb" -// RVA23S64: "-target-feature" "+zbs" -// RVA23S64: "-target-feature" "+zkt" -// RVA23S64: "-target-feature" "+zvbb" -// RVA23S64: "-target-feature" "+zvfhmin" -// RVA23S64: "-target-feature" "+zvkt" -// RVA23S64: "-target-feature" "+shcounterenw" -// RVA23S64: "-target-feature" "+shgatpa" -// RVA23S64: "-target-feature" "+shtvala" -// RVA23S64: "-target-feature" "+shvsatpa" -// RVA23S64: "-target-feature" "+shvstvala" -// RVA23S64: "-target-feature" "+shvstvecd" -// RVA23S64: "-target-feature" "+ssccptr" -// RVA23S64: "-target-feature" "+sscofpmf" -// RVA23S64: "-target-feature" "+sscounterenw" -// RVA23S64: "-target-feature" "+experimental-ssnpm" -// RVA23S64: "-target-feature" "+ssstateen" -// RVA23S64: "-target-feature" "+sstc" -// RVA23S64: "-target-feature" "+sstvala" -// RVA23S64: "-target-feature" "+sstvecd" -// RVA23S64: "-target-feature" "+ssu64xl" -// RVA23S64: "-target-feature" "+svade" -// RVA23S64: "-target-feature" "+svbare" -// RVA23S64: "-target-feature" "+svinval" -// RVA23S64: "-target-feature" "+svnapot" -// RVA23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s -// RVB23U64: "-target-feature" "+m" -// RVB23U64: "-target-feature" "+a" -// RVB23U64: "-target-feature" "+f" -// RVB23U64: "-target-feature" "+d" -// RVB23U64: "-target-feature" "+c" -// RVB23U64: "-target-feature" "+zic64b" -// RVB23U64: "-target-feature" "+zicbom" -// RVB23U64: "-target-feature" "+zicbop" -// RVB23U64: "-target-feature" "+zicboz" -// RVB23U64: "-target-feature" "+ziccamoa" -// RVB23U64: "-target-feature" "+ziccif" -// RVB23U64: "-target-feature" "+zicclsm" -// RVB23U64: "-target-feature" "+ziccrse" -// RVB23U64: "-target-feature" "+zicntr" -// RVB23U64: "-target-feature" "+zicond" -// RVB23U64: "-target-feature" "+zicsr" -// RVB23U64: "-target-feature" "+zihintntl" -// RVB23U64: "-target-feature" "+zihintpause" -// RVB23U64: "-target-feature" "+zihpm" -// RVB23U64: "-target-feature" "+experimental-zimop" -// RVB23U64: "-target-feature" "+za64rs" -// RVB23U64: "-target-feature" "+zawrs" -// RVB23U64: "-target-feature" "+zfa" -// RVB23U64: "-target-feature" "+zcb" -// RVB23U64: "-target-feature" "+experimental-zcmop" -// RVB23U64: "-target-feature" "+zba" -// RVB23U64: "-target-feature" "+zbb" -// RVB23U64: "-target-feature" "+zbs" -// RVB23U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s -// RVB23S64: "-target-feature" "+m" -// RVB23S64: "-target-feature" "+a" -// RVB23S64: "-target-feature" "+f" -// RVB23S64: "-target-feature" "+d" -// RVB23S64: "-target-feature" "+c" -// RVB23S64: "-target-feature" "+zic64b" -// RVB23S64: "-target-feature" "+zicbom" -// RVB23S64: "-target-feature" "+zicbop" -// RVB23S64: "-target-feature" "+zicboz" -// RVB23S64: "-target-feature" "+ziccamoa" -// RVB23S64: "-target-feature" "+ziccif" -// RVB23S64: "-target-feature" "+zicclsm" -// RVB23S64: "-target-feature" "+ziccrse" -// RVB23S64: "-target-feature" "+zicntr" -// RVB23S64: "-target-feature" "+zicond" -// RVB23S64: "-target-feature" "+zicsr" -// RVB23S64: "-target-feature" "+zifencei" -// RVB23S64: "-target-feature" "+zihintntl" -// RVB23S64: "-target-feature" "+zihintpause" -// RVB23S64: "-target-feature" "+zihpm" -// RVB23S64: "-target-feature" "+experimental-zimop" -// RVB23S64: "-target-feature" "+za64rs" -// RVB23S64: "-target-feature" "+zawrs" -// RVB23S64: "-target-feature" "+zfa" -// RVB23S64: "-target-feature" "+zcb" -// RVB23S64: "-target-feature" "+experimental-zcmop" -// RVB23S64: "-target-feature" "+zba" -// RVB23S64: "-target-feature" "+zbb" -// RVB23S64: "-target-feature" "+zbs" -// RVB23S64: "-target-feature" "+zkt" -// RVB23S64: "-target-feature" "+ssccptr" -// RVB23S64: "-target-feature" "+sscofpmf" -// RVB23S64: "-target-feature" "+sscounterenw" -// RVB23S64: "-target-feature" "+sstc" -// RVB23S64: "-target-feature" "+sstvala" -// RVB23S64: "-target-feature" "+sstvecd" -// RVB23S64: "-target-feature" "+ssu64xl" -// RVB23S64: "-target-feature" "+svade" -// RVB23S64: "-target-feature" "+svbare" -// RVB23S64: "-target-feature" "+svinval" -// RVB23S64: "-target-feature" "+svnapot" -// RVB23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s -// RVM23U32: "-target-feature" "+m" -// RVM23U32: "-target-feature" "+zicbop" -// RVM23U32: "-target-feature" "+zicond" -// RVM23U32: "-target-feature" "+zicsr" -// RVM23U32: "-target-feature" "+zihintntl" -// RVM23U32: "-target-feature" "+zihintpause" -// RVM23U32: "-target-feature" "+experimental-zimop" -// RVM23U32: "-target-feature" "+zce" -// RVM23U32: "-target-feature" "+experimental-zcmop" -// RVM23U32: "-target-feature" "+zba" -// RVM23U32: "-target-feature" "+zbb" -// RVM23U32: "-target-feature" "+zbs" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" - -// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s -// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile - -// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s -// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 67e6e5b962b1..39235ace4724 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,11 +36,6 @@ struct RISCVSupportedExtension { } }; -struct RISCVProfile { - StringLiteral Name; - StringLiteral MArch; -}; - } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -249,42 +244,6 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on -static constexpr RISCVProfile SupportedProfiles[] = { - {"rvi20u32", "rv32i"}, - {"rvi20u64", "rv64i"}, - {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, - {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" - "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, - {"rva22u64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, - {"rva22s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" - "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, - {"rva23u64", - "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" - "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, - {"rva23s64", - "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" - "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" - "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" - "svbare_svinval_svnapot_svpbmt"}, - {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" - "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" - "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, - {"rvb23s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" - "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, - {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" - "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, -}; - static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -898,29 +857,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } - if (Arch.starts_with("rvi") || Arch.starts_with("rva") || - Arch.starts_with("rvb") || Arch.starts_with("rvm")) { - const auto *FoundProfile = - llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { - return Arch.starts_with(Profile.Name); - }); - - if (FoundProfile == std::end(SupportedProfiles)) - return createStringError(errc::invalid_argument, "unsupported profile"); - - std::string NewArch = FoundProfile->MArch.str(); - StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); - if (!ArchWithoutProfile.empty()) { - if (!ArchWithoutProfile.starts_with("_")) - return createStringError( - errc::invalid_argument, - "additional extensions must be after separator '_'"); - NewArch += ArchWithoutProfile.str(); - } - return parseArchString(NewArch, EnableExperimentalExtension, - ExperimentalExtensionVersionCheck, IgnoreUnknown); - } - bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From a62441d4bb6bd0cd8eccab8c5692340c5a2c60bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 141/933] [clang][analyzer][NFC] UnixAPIMisuseChecker inherits from Checker (#83027) --- .../Checkers/UnixAPIChecker.cpp | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 19f1ca2dc824..599e5e6cedc6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -17,6 +17,7 @@ #include "clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h" #include "clang/StaticAnalyzer/Core/Checker.h" #include "clang/StaticAnalyzer/Core/CheckerManager.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" #include "llvm/ADT/STLExtras.h" @@ -41,8 +42,7 @@ enum class OpenVariant { namespace { class UnixAPIMisuseChecker - : public Checker, - check::ASTDecl> { + : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; @@ -52,14 +52,14 @@ public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; - void checkPreStmt(const CallExpr *CE, CheckerContext &C) const; + void checkPreCall(const CallEvent &Call, CheckerContext &C) const; - void CheckOpen(CheckerContext &C, const CallExpr *CE) const; - void CheckOpenAt(CheckerContext &C, const CallExpr *CE) const; - void CheckPthreadOnce(CheckerContext &C, const CallExpr *CE) const; + void CheckOpen(CheckerContext &C, const CallEvent &Call) const; + void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; - void CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, OpenVariant Variant) const; + void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, + OpenVariant Variant) const; void ReportOpenBug(CheckerContext &C, ProgramStateRef State, const char *Msg, SourceRange SR) const; @@ -113,9 +113,9 @@ void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, // "open" (man 2 open) //===----------------------------------------------------------------------===/ -void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, +void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, CheckerContext &C) const { - const FunctionDecl *FD = C.getCalleeDecl(CE); + const FunctionDecl *FD = dyn_cast_if_present(Call.getDecl()); if (!FD || FD->getKind() != Decl::Function) return; @@ -130,13 +130,13 @@ void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, return; if (FName == "open") - CheckOpen(C, CE); + CheckOpen(C, Call); else if (FName == "openat") - CheckOpenAt(C, CE); + CheckOpenAt(C, Call); else if (FName == "pthread_once") - CheckPthreadOnce(C, CE); + CheckPthreadOnce(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -152,17 +152,17 @@ void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, } void UnixAPIMisuseChecker::CheckOpen(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::Open); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::Open); } void UnixAPIMisuseChecker::CheckOpenAt(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::OpenAt); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::OpenAt); } void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, + const CallEvent &Call, OpenVariant Variant) const { // The index of the argument taking the flags open flags (O_RDONLY, // O_WRONLY, O_CREAT, etc.), @@ -191,11 +191,11 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ProgramStateRef state = C.getState(); - if (CE->getNumArgs() < MinArgCount) { + if (Call.getNumArgs() < MinArgCount) { // The frontend should issue a warning for this case. Just return. return; - } else if (CE->getNumArgs() == MaxArgCount) { - const Expr *Arg = CE->getArg(CreateModeArgIndex); + } else if (Call.getNumArgs() == MaxArgCount) { + const Expr *Arg = Call.getArgExpr(CreateModeArgIndex); QualType QT = Arg->getType(); if (!QT->isIntegerType()) { SmallString<256> SBuf; @@ -209,7 +209,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, Arg->getSourceRange()); return; } - } else if (CE->getNumArgs() > MaxArgCount) { + } else if (Call.getNumArgs() > MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' with more than " << MaxArgCount @@ -217,7 +217,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ReportOpenBug(C, state, SBuf.c_str(), - CE->getArg(MaxArgCount)->getSourceRange()); + Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -226,8 +226,8 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } // Now check if oflags has O_CREAT set. - const Expr *oflagsEx = CE->getArg(FlagsArgIndex); - const SVal V = C.getSVal(oflagsEx); + const Expr *oflagsEx = Call.getArgExpr(FlagsArgIndex); + const SVal V = Call.getArgSVal(FlagsArgIndex); if (!isa(V)) { // The case where 'V' can be a location can only be due to a bad header, // so in this case bail out. @@ -253,7 +253,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, if (!(trueState && !falseState)) return; - if (CE->getNumArgs() < MaxArgCount) { + if (Call.getNumArgs() < MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' requires a " @@ -271,18 +271,18 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, //===----------------------------------------------------------------------===// void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, - const CallExpr *CE) const { + const CallEvent &Call) const { // This is similar to 'CheckDispatchOnce' in the MacOSXAPIChecker. // They can possibly be refactored. - if (CE->getNumArgs() < 1) + if (Call.getNumArgs() < 1) return; // Check if the first argument is stack allocated. If so, issue a warning // because that's likely to be bad news. ProgramStateRef state = C.getState(); - const MemRegion *R = C.getSVal(CE->getArg(0)).getAsRegion(); + const MemRegion *R = Call.getArgSVal(0).getAsRegion(); if (!R || !isa(R->getMemorySpace())) return; @@ -304,7 +304,7 @@ void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, auto report = std::make_unique(BT_pthreadOnce, os.str(), N); - report->addRange(CE->getArg(0)->getSourceRange()); + report->addRange(Call.getArgExpr(0)->getSourceRange()); C.emitReport(std::move(report)); } -- GitLab From 730ca47a0cc7380def6df1d25b30c1378fd8bf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 142/933] [clang][analyzer] Model getline/getdelim preconditions and evaluation (#83027) According to POSIX 2018. 1. lineptr, n and stream can not be NULL. 2. If *n is non-zero, *lineptr must point to a region of at least *n bytes, or be a NULL pointer. Additionally, if *lineptr is not NULL, *n must not be undefined. --- .../Core/PathSensitive/CheckerHelpers.h | 4 +- .../StaticAnalyzer/Checkers/MallocChecker.cpp | 8 +- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 23 +- .../Checkers/UnixAPIChecker.cpp | 132 ++++++- .../StaticAnalyzer/Core/CheckerHelpers.cpp | 5 +- clang/test/Analysis/getline-unixapi.c | 322 ++++++++++++++++++ clang/test/Analysis/stream.c | 73 ++++ 7 files changed, 554 insertions(+), 13 deletions(-) create mode 100644 clang/test/Analysis/getline-unixapi.c diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h index 60421e5437d8..d053a9718912 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h @@ -15,7 +15,6 @@ #include "ProgramState_Fwd.h" #include "SVals.h" - #include "clang/AST/OperationKinds.h" #include "clang/AST/Stmt.h" #include "clang/Basic/OperatorKinds.h" @@ -113,8 +112,7 @@ public: OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, bool IsBinary); -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State); +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State); } // namespace ento diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp index 03cb7696707f..c2d96f592609 100644 --- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp @@ -1441,7 +1441,7 @@ void MallocChecker::preGetdelim(const CallEvent &Call, return; ProgramStateRef State = C.getState(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); + const auto LinePtr = getPointeeVal(Call.getArgSVal(0), State); if (!LinePtr) return; @@ -1470,8 +1470,10 @@ void MallocChecker::checkGetdelim(const CallEvent &Call, SValBuilder &SVB = C.getSValBuilder(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); - const auto Size = getPointeeDefVal(Call.getArgSVal(1), State); + const auto LinePtr = + getPointeeVal(Call.getArgSVal(0), State)->getAs(); + const auto Size = + getPointeeVal(Call.getArgSVal(1), State)->getAs(); if (!LinePtr || !Size || !LinePtr->getAsRegion()) return; diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 10972158f398..902c42a2799b 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -1200,10 +1200,25 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, // Add transition for the successful state. NonLoc RetVal = makeRetVal(C, E.CE).castAs(); - ProgramStateRef StateNotFailed = - State->BindExpr(E.CE, C.getLocationContext(), RetVal); + ProgramStateRef StateNotFailed = E.bindReturnValue(State, C, RetVal); StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GE, RetVal, E.getZeroVal(Call)); + + // On success, a buffer is allocated. + auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State); + if (NewLinePtr && isa(*NewLinePtr)) + StateNotFailed = StateNotFailed->assume( + NewLinePtr->castAs(), true); + + // The buffer size `*n` must be enough to hold the whole line, and + // greater than the return value, since it has to account for '\0'. + SVal SizePtrSval = Call.getArgSVal(1); + auto NVal = getPointeeVal(SizePtrSval, State); + if (NVal && isa(*NVal)) { + StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GT, + NVal->castAs(), RetVal); + StateNotFailed = E.bindReturnValue(StateNotFailed, C, RetVal); + } if (!StateNotFailed) return; C.addTransition(StateNotFailed); @@ -1217,6 +1232,10 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, E.isStreamEof() ? ErrorFEof : ErrorFEof | ErrorFError; StateFailed = E.setStreamState( StateFailed, StreamState::getOpened(Desc, NewES, !NewES.isFEof())); + // On failure, the content of the buffer is undefined. + if (auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State)) + StateFailed = StateFailed->bindLoc(*NewLinePtr, UndefinedVal(), + C.getLocationContext()); C.addTransition(StateFailed, E.getFailureNoteTag(this, C)); } diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 599e5e6cedc6..da2d16ca9b5d 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -20,6 +20,7 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" @@ -44,10 +45,23 @@ namespace { class UnixAPIMisuseChecker : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; + const BugType BT_getline{this, "Improper use of getdelim", + categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; + const BugType BT_ArgumentNull{this, "NULL pointer", categories::UnixAPI}; mutable std::optional Val_O_CREAT; + ProgramStateRef + EnsurePtrNotNull(SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, + ProgramStateRef State, const StringRef PtrDescr, + std::optional> BT = + std::nullopt) const; + + ProgramStateRef EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const; + public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; @@ -56,6 +70,7 @@ public: void CheckOpen(CheckerContext &C, const CallEvent &Call) const; void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckGetDelim(CheckerContext &C, const CallEvent &Call) const; void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, @@ -95,6 +110,30 @@ private: } // end anonymous namespace +ProgramStateRef UnixAPIMisuseChecker::EnsurePtrNotNull( + SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, ProgramStateRef State, + const StringRef PtrDescr, + std::optional> BT) const { + const auto Ptr = PtrVal.getAs(); + if (!Ptr) + return State; + + const auto [PtrNotNull, PtrNull] = State->assume(*Ptr); + if (!PtrNotNull && PtrNull) { + if (ExplodedNode *N = C.generateErrorNode(PtrNull)) { + auto R = std::make_unique( + BT.value_or(std::cref(BT_ArgumentNull)), + (PtrDescr + " pointer might be NULL.").str(), N); + if (PtrExpr) + bugreporter::trackExpressionValue(N, PtrExpr, *R); + C.emitReport(std::move(R)); + } + return nullptr; + } + + return PtrNotNull; +} + void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &) const { @@ -137,6 +176,9 @@ void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, else if (FName == "pthread_once") CheckPthreadOnce(C, Call); + + else if (is_contained({"getdelim", "getline"}, FName)) + CheckGetDelim(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -215,8 +257,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, OS << "Call to '" << VariantName << "' with more than " << MaxArgCount << " arguments"; - ReportOpenBug(C, state, - SBuf.c_str(), + ReportOpenBug(C, state, SBuf.c_str(), Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -266,6 +307,93 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } } +//===----------------------------------------------------------------------===// +// getdelim and getline +//===----------------------------------------------------------------------===// + +ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const { + static constexpr llvm::StringLiteral SizeGreaterThanBufferSize = + "The buffer from the first argument is smaller than the size " + "specified by the second parameter"; + static constexpr llvm::StringLiteral SizeUndef = + "The buffer from the first argument is not NULL, but the size specified " + "by the second parameter is undefined."; + + auto EmitBugReport = [this, &C, SizePtrExpr, LinePtrPtrExpr]( + ProgramStateRef BugState, StringRef ErrMsg) { + if (ExplodedNode *N = C.generateErrorNode(BugState)) { + auto R = std::make_unique(BT_getline, ErrMsg, N); + bugreporter::trackExpressionValue(N, SizePtrExpr, *R); + bugreporter::trackExpressionValue(N, LinePtrPtrExpr, *R); + C.emitReport(std::move(R)); + } + }; + + // We have a pointer to a pointer to the buffer, and a pointer to the size. + // We want what they point at. + auto LinePtrSVal = getPointeeVal(LinePtrPtrSVal, State)->getAs(); + auto NSVal = getPointeeVal(SizePtrSVal, State); + if (!LinePtrSVal || !NSVal || NSVal->isUnknown()) + return nullptr; + + assert(LinePtrPtrExpr && SizePtrExpr); + + const auto [LinePtrNotNull, LinePtrNull] = State->assume(*LinePtrSVal); + if (LinePtrNotNull && !LinePtrNull) { + // If `*lineptr` is not null, but `*n` is undefined, there is UB. + if (NSVal->isUndef()) { + EmitBugReport(LinePtrNotNull, SizeUndef); + return nullptr; + } + + // If it is defined, and known, its size must be less than or equal to + // the buffer size. + auto NDefSVal = NSVal->getAs(); + auto &SVB = C.getSValBuilder(); + auto LineBufSize = + getDynamicExtent(LinePtrNotNull, LinePtrSVal->getAsRegion(), SVB); + auto LineBufSizeGtN = SVB.evalBinOp(LinePtrNotNull, BO_GE, LineBufSize, + *NDefSVal, SVB.getConditionType()) + .getAs(); + if (!LineBufSizeGtN) + return LinePtrNotNull; + if (auto LineBufSizeOk = LinePtrNotNull->assume(*LineBufSizeGtN, true)) + return LineBufSizeOk; + + EmitBugReport(LinePtrNotNull, SizeGreaterThanBufferSize); + return nullptr; + } + return State; +} + +void UnixAPIMisuseChecker::CheckGetDelim(CheckerContext &C, + const CallEvent &Call) const { + ProgramStateRef State = C.getState(); + + // The parameter `n` must not be NULL. + SVal SizePtrSval = Call.getArgSVal(1); + State = EnsurePtrNotNull(SizePtrSval, Call.getArgExpr(1), C, State, "Size"); + if (!State) + return; + + // The parameter `lineptr` must not be NULL. + SVal LinePtrPtrSVal = Call.getArgSVal(0); + State = + EnsurePtrNotNull(LinePtrPtrSVal, Call.getArgExpr(0), C, State, "Line"); + if (!State) + return; + + State = EnsureGetdelimBufferAndSizeCorrect(LinePtrPtrSVal, SizePtrSval, + Call.getArgExpr(0), + Call.getArgExpr(1), C, State); + if (!State) + return; + + C.addTransition(State); +} + //===----------------------------------------------------------------------===// // pthread_once //===----------------------------------------------------------------------===// diff --git a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp index 364c87e910b7..d7137a915b3d 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp @@ -183,10 +183,9 @@ OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, } } -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State) { +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State) { if (const auto *Ptr = PtrSVal.getAsRegion()) { - return State->getSVal(Ptr).getAs(); + return State->getSVal(Ptr); } return std::nullopt; } diff --git a/clang/test/Analysis/getline-unixapi.c b/clang/test/Analysis/getline-unixapi.c new file mode 100644 index 000000000000..86635ed84997 --- /dev/null +++ b/clang/test/Analysis/getline-unixapi.c @@ -0,0 +1,322 @@ +// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,debug.ExprInspection -verify %s + +#include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" +#include "Inputs/system-header-simulator-for-valist.h" + +void clang_analyzer_eval(int); +void clang_analyzer_dump_int(int); +void clang_analyzer_dump_ptr(void*); +void clang_analyzer_warnIfReached(); + +void test_getline_null_lineptr() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char **buffer = NULL; + size_t n = 0; + getline(buffer, &n, F1); // expected-warning {{Line pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getline(&buffer, NULL, F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getline(&buffer, &n, F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getline(&buffer, &n, F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + + getline(&buffer, &n, F1); // ok + fclose(F1); + free(buffer); +} + +void test_getline_null_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = NULL; + size_t n; + + getline(&buffer, &n, F1); // ok since posix 2018 + fclose(F1); + free(buffer); +} + +void test_getline_buffer_size_0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 0; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, the buffer is enough for 0 character + fclose(F1); + free(buffer); +} + +void test_getline_buffer_bad_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 100; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is smaller than the size specified by the second parameter}} + fclose(F1); + free(buffer); +} + +void test_getline_buffer_smaller_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n = 10; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, there is enough space for 10 characters + fclose(F1); + free(buffer); +} + +void test_getline_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is not NULL, but the size specified by the second parameter is undefined}} + fclose(F1); + free(buffer); +} + + +void test_getline_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getline(&buffer, &n, F1); + // getline returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, NULL, ',', F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getdelim(&buffer, &n, ';', F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getdelim(&buffer, &n, ' ', F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, &n, '-', F1); // ok + fclose(F1); + free(buffer); +} + +void test_getdelim_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getdelim(&buffer, &n, '\r', F1); + // getdelim returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } + else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getline_while() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t read; + + while ((read = getline(&line, &len, file)) != -1) { + printf("%s\n", line); + } + + free(line); + fclose(file); +} + +void test_getline_return_check() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t r = getline(&line, &len, file); + + if (r != -1) { + if (line[0] == '\0') {} // ok + } + free(line); + fclose(file); +} + +void test_getline_clear_eof() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 10; + char *buffer = malloc(n); + ssize_t read = fread(buffer, n, 1, file); + if (feof(file)) { + clearerr(file); + getline(&buffer, &n, file); // ok + } + fclose(file); + free(buffer); +} + +void test_getline_not_null(char **buffer, size_t *size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + getline(buffer, size, file); + fclose(file); + + if (size == NULL || buffer == NULL) { + clang_analyzer_warnIfReached(); // must not happen + } +} + +void test_getline_size_constraint(size_t size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t old_size = size; + char *buffer = malloc(10); + if (buffer != NULL) { + ssize_t r = getline(&buffer, &size, file); + if (r >= 0) { + // Since buffer has a size of 10, old_size must be less than or equal to 10. + // Otherwise, there would be UB. + clang_analyzer_eval(old_size <= 10); // expected-warning{{TRUE}} + } + } + fclose(file); + free(buffer); +} + +void test_getline_negative_buffer() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *buffer = NULL; + size_t n = -1; + getline(&buffer, &n, file); // ok since posix 2018 + free(buffer); + fclose(file); +} + +void test_getline_negative_buffer_2(char *buffer) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + (void)getline(&buffer, &n, file); // ok + free(buffer); + fclose(file); +} diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c index 7ba27740a937..ba5e66a4102e 100644 --- a/clang/test/Analysis/stream.c +++ b/clang/test/Analysis/stream.c @@ -4,6 +4,7 @@ // RUN: %clang_analyze_cc1 -triple=hexagon -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s #include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" #include "Inputs/system-header-simulator-for-valist.h" void clang_analyzer_eval(int); @@ -376,3 +377,75 @@ void fflush_on_open_failed_stream(void) { } fclose(F); } + +void getline_null_file() { + char *buffer = NULL; + size_t n = 0; + getline(&buffer, &n, NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getdelim_null_file() { + char *buffer = NULL; + size_t n = 0; + getdelim(&buffer, &n, '\n', NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getline_buffer_on_error() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + if (getline(&line, &len, file) == -1) { + if (line[0] == '\0') {} // expected-warning {{The left operand of '==' is a garbage value}} + } else { + if (line[0] == '\0') {} // no warning + } + + free(line); + fclose(file); +} + +void getline_ret_value() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 0; + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + // The return value does *not* include the terminating null byte. + // The buffer must be large enough to include it. + clang_analyzer_eval(n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + fclose(file); + free(buffer); +} + + +void getline_buffer_size_negative() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + clang_analyzer_eval((ssize_t)n >= 0); // expected-warning{{FALSE}} + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + clang_analyzer_eval((ssize_t)n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + free(buffer); + fclose(file); +} -- GitLab From d8e5c0b4e546c73b2d10956a9517f1f2727702ae Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:01:01 -0400 Subject: [PATCH 143/933] [DXIL] Complete abs lowering (#86158) This change completes #86155 - `DXIL.td` - lowering `fabs` intrinsic to the float dxil op. - `DXILIntrinsicExpansion.cpp` - Add intrinsic expansion for the abs case. --- llvm/lib/Target/DirectX/DXIL.td | 2 + .../Target/DirectX/DXILIntrinsicExpansion.cpp | 23 +++++++++++ llvm/test/CodeGen/DirectX/abs-vec.ll | 34 +++++++++++++++++ llvm/test/CodeGen/DirectX/abs.ll | 38 +++++++++++++++++++ llvm/test/CodeGen/DirectX/fabs.ll | 32 ++++++++++++++++ 5 files changed, 129 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/abs-vec.ll create mode 100644 llvm/test/CodeGen/DirectX/abs.ll create mode 100644 llvm/test/CodeGen/DirectX/fabs.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 36eb29d53766..ba7cd6ae9183 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -255,6 +255,8 @@ class DXILOpMapping; def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp index 0db42bc0a0fb..b46564702c7a 100644 --- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp +++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp @@ -33,6 +33,7 @@ using namespace llvm; static bool isIntrinsicExpansion(Function &F) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: case Intrinsic::exp: case Intrinsic::dx_any: case Intrinsic::dx_clamp: @@ -46,6 +47,26 @@ static bool isIntrinsicExpansion(Function &F) { return false; } +static bool expandAbs(CallInst *Orig) { + Value *X = Orig->getOperand(0); + IRBuilder<> Builder(Orig->getParent()); + Builder.SetInsertPoint(Orig); + Type *Ty = X->getType(); + Type *EltTy = Ty->getScalarType(); + Constant *Zero = Ty->isVectorTy() + ? ConstantVector::getSplat( + ElementCount::getFixed( + cast(Ty)->getNumElements()), + ConstantInt::get(EltTy, 0)) + : ConstantInt::get(EltTy, 0); + auto *V = Builder.CreateSub(Zero, X); + auto *MaxCall = + Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr, "dx.max"); + Orig->replaceAllUsesWith(MaxCall); + Orig->eraseFromParent(); + return true; +} + static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) { assert(DotIntrinsic == Intrinsic::dx_sdot || DotIntrinsic == Intrinsic::dx_udot); @@ -213,6 +234,8 @@ static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) { static bool expandIntrinsic(Function &F, CallInst *Orig) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: + return expandAbs(Orig); case Intrinsic::exp: return expandExpIntrinsic(Orig); case Intrinsic::dx_any: diff --git a/llvm/test/CodeGen/DirectX/abs-vec.ll b/llvm/test/CodeGen/DirectX/abs-vec.ll new file mode 100644 index 000000000000..1c40555eb390 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs-vec.ll @@ -0,0 +1,34 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for int vectors. + +; CHECK-LABEL: abs_i16Vec2 +define noundef <2 x i16> @abs_i16Vec2(<2 x i16> noundef %a) #0 { +entry: +; CHECK: sub <2 x i16> zeroinitializer, %a +; CHECK: call <2 x i16> @llvm.smax.v2i16(<2 x i16> %a, <2 x i16> %{{.*}}) + %elt.abs = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %a, i1 false) + ret <2 x i16> %elt.abs +} + +; CHECK-LABEL: abs_i32Vec3 +define noundef <3 x i32> @abs_i32Vec3(<3 x i32> noundef %a) #0 { +entry: +; CHECK: sub <3 x i32> zeroinitializer, %a +; CHECK: call <3 x i32> @llvm.smax.v3i32(<3 x i32> %a, <3 x i32> %{{.*}}) + %elt.abs = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 false) + ret <3 x i32> %elt.abs +} + +; CHECK-LABEL: abs_i64Vec4 +define noundef <4 x i64> @abs_i64Vec4(<4 x i64> noundef %a) #0 { +entry: +; CHECK: sub <4 x i64> zeroinitializer, %a +; CHECK: call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %{{.*}}) + %elt.abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a, i1 false) + ret <4 x i64> %elt.abs +} + +declare <2 x i16> @llvm.abs.v2i16(<2 x i16>, i1 immarg) +declare <3 x i32> @llvm.abs.v3i32(<3 x i32>, i1 immarg) +declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/abs.ll b/llvm/test/CodeGen/DirectX/abs.ll new file mode 100644 index 000000000000..822580e8c089 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs.ll @@ -0,0 +1,38 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK + +; Make sure dxil operation function calls for abs are generated for int16_t/int/int64_t. + +; CHECK-LABEL: abs_i16 +define noundef i16 @abs_i16(i16 noundef %a) { +entry: +; CHECK: sub i16 0, %a +; EXPCHECK: call i16 @llvm.smax.i16(i16 %a, i16 %{{.*}}) +; DOPCHECK: call i16 @dx.op.binary.i16(i32 37, i16 %a, i16 %{{.*}}) + %elt.abs = call i16 @llvm.abs.i16(i16 %a, i1 false) + ret i16 %elt.abs +} + +; CHECK-LABEL: abs_i32 +define noundef i32 @abs_i32(i32 noundef %a) { +entry: +; CHECK: sub i32 0, %a +; EXPCHECK: call i32 @llvm.smax.i32(i32 %a, i32 %{{.*}}) +; DOPCHECK: call i32 @dx.op.binary.i32(i32 37, i32 %a, i32 %{{.*}}) + %elt.abs = call i32 @llvm.abs.i32(i32 %a, i1 false) + ret i32 %elt.abs +} + +; CHECK-LABEL: abs_i64 +define noundef i64 @abs_i64(i64 noundef %a) { +entry: +; CHECK: sub i64 0, %a +; EXPCHECK: call i64 @llvm.smax.i64(i64 %a, i64 %{{.*}}) +; DOPCHECK: call i64 @dx.op.binary.i64(i32 37, i64 %a, i64 %{{.*}}) + %elt.abs = call i64 @llvm.abs.i64(i64 %a, i1 false) + ret i64 %elt.abs +} + +declare i16 @llvm.abs.i16(i16, i1 immarg) +declare i32 @llvm.abs.i32(i32, i1 immarg) +declare i64 @llvm.abs.i64(i64, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/fabs.ll b/llvm/test/CodeGen/DirectX/fabs.ll new file mode 100644 index 000000000000..3b3f8aa9a4a9 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/fabs.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for float, half, and double. + + +; CHECK-LABEL: fabs_half +define noundef half @fabs_half(half noundef %a) { +entry: + ; CHECK: call half @dx.op.unary.f16(i32 6, half %{{.*}}) + %elt.abs = call half @llvm.fabs.f16(half %a) + ret half %elt.abs +} + +; CHECK-LABEL: fabs_float +define noundef float @fabs_float(float noundef %a) { +entry: +; CHECK: call float @dx.op.unary.f32(i32 6, float %{{.*}}) + %elt.abs = call float @llvm.fabs.f32(float %a) + ret float %elt.abs +} + +; CHECK-LABEL: fabs_double +define noundef double @fabs_double(double noundef %a) { +entry: +; CHECK: call double @dx.op.unary.f64(i32 6, double %{{.*}}) + %elt.abs = call double @llvm.fabs.f64(double %a) + ret double %elt.abs +} + +declare half @llvm.fabs.f16(half) +declare float @llvm.fabs.f32(float) +declare double @llvm.fabs.f64(double) -- GitLab From 79c32eb03d9ee4dd0a913c4130bc87c5e8ce7908 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:02:47 -0400 Subject: [PATCH 144/933] [DXIL] Add lowerings for cosine and floor (#86173) Completes #86170 Completes #86172 - `DXIL.td` - Add changes to lower the cosine and floor intrinsics to dxilOps. --- llvm/lib/Target/DirectX/DXIL.td | 6 ++++++ llvm/test/CodeGen/DirectX/cos.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/cos_error.ll | 10 ++++++++++ llvm/test/CodeGen/DirectX/floor.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/floor_error.ll | 10 ++++++++++ 5 files changed, 66 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/cos.ll create mode 100644 llvm/test/CodeGen/DirectX/cos_error.ll create mode 100644 llvm/test/CodeGen/DirectX/floor.ll create mode 100644 llvm/test/CodeGen/DirectX/floor_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index ba7cd6ae9183..f7e69ebae15b 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -260,6 +260,9 @@ def Abs : DXILOpMapping<6, unary, int_fabs, def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; +def Cos : DXILOpMapping<12, unary, int_cos, + "Returns cosine(theta) for theta in radians.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Sin : DXILOpMapping<13, unary, int_sin, "Returns sine(theta) for theta in radians.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; @@ -279,6 +282,9 @@ def Round : DXILOpMapping<26, unary, int_round, "Returns the input rounded to the nearest integer" "within a floating-point type.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; +def Floor : DXILOpMapping<27, unary, int_floor, + "Returns the largest integer that is less than or equal to the input.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def FMax : DXILOpMapping<35, binary, int_maxnum, "Float maximum. FMax(a,b) = a > b ? a : b">; def FMin : DXILOpMapping<36, binary, int_minnum, diff --git a/llvm/test/CodeGen/DirectX/cos.ll b/llvm/test/CodeGen/DirectX/cos.ll new file mode 100644 index 000000000000..00f2e2c3f6e5 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for cos are generated for float and half. + +define noundef float @cos_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 12, float %{{.*}}) + %elt.cos = call float @llvm.cos.f32(float %a) + ret float %elt.cos +} + +define noundef half @cos_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 12, half %{{.*}}) + %elt.cos = call half @llvm.cos.f16(half %a) + ret half %elt.cos +} + +declare half @llvm.cos.f16(half) +declare float @llvm.cos.f32(float) diff --git a/llvm/test/CodeGen/DirectX/cos_error.ll b/llvm/test/CodeGen/DirectX/cos_error.ll new file mode 100644 index 000000000000..a074f5b493df --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation cos does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @cos_double(double noundef %a) { +entry: + %elt.cos = call double @llvm.cos.f64(double %a) + ret double %elt.cos +} diff --git a/llvm/test/CodeGen/DirectX/floor.ll b/llvm/test/CodeGen/DirectX/floor.ll new file mode 100644 index 000000000000..b033e2eaa491 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for floor are generated for float and half. + +define noundef float @floor_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 27, float %{{.*}}) + %elt.floor = call float @llvm.floor.f32(float %a) + ret float %elt.floor +} + +define noundef half @floor_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 27, half %{{.*}}) + %elt.floor = call half @llvm.floor.f16(half %a) + ret half %elt.floor +} + +declare half @llvm.floor.f16(half) +declare float @llvm.floor.f32(float) diff --git a/llvm/test/CodeGen/DirectX/floor_error.ll b/llvm/test/CodeGen/DirectX/floor_error.ll new file mode 100644 index 000000000000..3b51a4b543b7 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation floor does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @floor_double(double noundef %a) { +entry: + %elt.floor = call double @llvm.floor.f64(double %a) + ret double %elt.floor +} -- GitLab From db33444ffa7e210e7040d8def958a14171f52eef Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Fri, 22 Mar 2024 12:03:28 +0100 Subject: [PATCH 145/933] CODEOWNERS: extend scope of MLIR transform dialect There are a bunch of related directories in another dialects. --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fea132c8fe78..77ba81c58c5d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -96,6 +96,7 @@ clang/test/AST/Interp/ @tbaederr # Transform Dialect in MLIR. /mlir/include/mlir/Dialect/Transform/* @ftynse @nicolasvasilache /mlir/lib/Dialect/Transform/* @ftynse @nicolasvasilache +/mlir/**/*TransformOps* @ftynse @nicolasvasilache # SPIR-V Dialect in MLIR. /mlir/**/SPIRV/ @antiagainst @kuhar -- GitLab From e925968e7815ac3810fdb54bb884b8a8bed02eb5 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Fri, 22 Mar 2024 12:04:44 +0100 Subject: [PATCH 146/933] [analyzer] Support C++23 static operator calls (#84972) Made by following: https://github.com/llvm/llvm-project/pull/83585#issuecomment-1980340866 Thanks for the details Tomek! CPP-5080 --- clang/docs/ReleaseNotes.rst | 1 + .../Core/PathSensitive/CallEvent.h | 72 +++++++++++++++++++ clang/lib/StaticAnalyzer/Core/CallEvent.cpp | 5 +- .../Core/ExprEngineCallAndReturn.cpp | 1 + clang/test/Analysis/cxx23-static-operator.cpp | 38 ++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 clang/test/Analysis/cxx23-static-operator.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..45b2e01af997 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -576,6 +576,7 @@ Static Analyzer - Fixed crashing on loops if the loop variable was declared in switch blocks but not under any case blocks if ``unroll-loops=true`` analyzer config is set. (#GH68819) +- Support C++23 static operator calls. (#GH84972) New features ^^^^^^^^^^^^ diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h index 0d36587484bf..549c864dc91e 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h @@ -59,6 +59,7 @@ namespace ento { enum CallEventKind { CE_Function, + CE_CXXStaticOperator, CE_CXXMember, CE_CXXMemberOperator, CE_CXXDestructor, @@ -709,6 +710,77 @@ public: } }; +/// Represents a static C++ operator call. +/// +/// "A" in this example. +/// However, "B" and "C" are represented by SimpleFunctionCall. +/// \code +/// struct S { +/// int pad; +/// static void operator()(int x, int y); +/// }; +/// S s{10}; +/// void (*fptr)(int, int) = &S::operator(); +/// +/// s(1, 2); // A +/// S::operator()(1, 2); // B +/// fptr(1, 2); // C +/// \endcode +class CXXStaticOperatorCall : public SimpleFunctionCall { + friend class CallEventManager; + +protected: + CXXStaticOperatorCall(const CXXOperatorCallExpr *CE, ProgramStateRef St, + const LocationContext *LCtx, + CFGBlock::ConstCFGElementRef ElemRef) + : SimpleFunctionCall(CE, St, LCtx, ElemRef) {} + CXXStaticOperatorCall(const CXXStaticOperatorCall &Other) = default; + + void cloneTo(void *Dest) const override { + new (Dest) CXXStaticOperatorCall(*this); + } + +public: + const CXXOperatorCallExpr *getOriginExpr() const override { + return cast(SimpleFunctionCall::getOriginExpr()); + } + + unsigned getNumArgs() const override { + // Ignore the object parameter that is not used for static member functions. + assert(getOriginExpr()->getNumArgs() > 0); + return getOriginExpr()->getNumArgs() - 1; + } + + const Expr *getArgExpr(unsigned Index) const override { + // Ignore the object parameter that is not used for static member functions. + return getOriginExpr()->getArg(Index + 1); + } + + std::optional + getAdjustedParameterIndex(unsigned ASTArgumentIndex) const override { + // Ignore the object parameter that is not used for static member functions. + if (ASTArgumentIndex == 0) + return std::nullopt; + return ASTArgumentIndex - 1; + } + + unsigned getASTArgumentIndex(unsigned CallArgumentIndex) const override { + // Account for the object parameter for the static member function. + return CallArgumentIndex + 1; + } + + OverloadedOperatorKind getOverloadedOperator() const { + return getOriginExpr()->getOperator(); + } + + Kind getKind() const override { return CE_CXXStaticOperator; } + StringRef getKindAsString() const override { return "CXXStaticOperatorCall"; } + + static bool classof(const CallEvent *CA) { + return CA->getKind() == CE_CXXStaticOperator; + } +}; + /// Represents a non-static C++ member function call. /// /// Example: \c obj.fun() diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp index bc14aea27f67..0e317ec765ec 100644 --- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -1408,9 +1408,12 @@ CallEventManager::getSimpleCall(const CallExpr *CE, ProgramStateRef State, if (const auto *OpCE = dyn_cast(CE)) { const FunctionDecl *DirectCallee = OpCE->getDirectCallee(); - if (const auto *MD = dyn_cast(DirectCallee)) + if (const auto *MD = dyn_cast(DirectCallee)) { if (MD->isImplicitObjectMemberFunction()) return create(OpCE, State, LCtx, ElemRef); + if (MD->isStatic()) + return create(OpCE, State, LCtx, ElemRef); + } } else if (CE->getCallee()->getType()->isBlockPointerType()) { return create(CE, State, LCtx, ElemRef); diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp index 4755b6bfa6dc..9d3e4fc944fb 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp @@ -846,6 +846,7 @@ ExprEngine::mayInlineCallKind(const CallEvent &Call, const ExplodedNode *Pred, const StackFrameContext *CallerSFC = CurLC->getStackFrame(); switch (Call.getKind()) { case CE_Function: + case CE_CXXStaticOperator: case CE_Block: break; case CE_CXXMember: diff --git a/clang/test/Analysis/cxx23-static-operator.cpp b/clang/test/Analysis/cxx23-static-operator.cpp new file mode 100644 index 000000000000..f380bd0dfa42 --- /dev/null +++ b/clang/test/Analysis/cxx23-static-operator.cpp @@ -0,0 +1,38 @@ +// RUN: %clang_analyze_cc1 -std=c++2b -verify %s \ +// RUN: -analyzer-checker=core,debug.ExprInspection + +template void clang_analyzer_dump(T); + +struct Adder { + int data; + static int operator()(int x, int y) { + clang_analyzer_dump(x); // expected-warning {{1}} + clang_analyzer_dump(y); // expected-warning {{2}} + return x + y; + } +}; + +void static_operator_call_inlines() { + Adder s{10}; + clang_analyzer_dump(s(1, 2)); // expected-warning {{3}} +} + +struct DataWithCtor { + int x; + int y; + DataWithCtor(int parm) : x(parm + 10), y(parm + 20) { + clang_analyzer_dump(this); // expected-warning {{&v}} + } +}; + +struct StaticSubscript { + static void operator[](DataWithCtor v) { + clang_analyzer_dump(v.x); // expected-warning {{20}} + clang_analyzer_dump(v.y); // expected-warning {{30}} + } +}; + +void top() { + StaticSubscript s; + s[DataWithCtor{10}]; +} -- GitLab From 74c3150ffc86a149abc68acdf8af1eed1ea0f038 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:23:25 +0000 Subject: [PATCH 147/933] [X86] Add shuffle tests from Issue #86076 SLP should be doing a better job, but both shuffles lower to poorer codegen than necessary --- .../CodeGen/X86/vector-shuffle-512-v16.ll | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll index dfa7f2dbdaee..c981d973fef3 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll @@ -177,6 +177,36 @@ define <16 x float> @shuffle_v16f32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28 ret <16 x float> %shuffle } +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15: +; ALL: # %bb.0: +; ALL-NEXT: vbroadcastss %xmm0, %ymm0 +; ALL-NEXT: vbroadcastss %xmm1, %ymm1 +; ALL-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %b0 = shufflevector <8 x float> %v0, <8 x float> poison, <8 x i32> zeroinitializer + %b1 = shufflevector <8 x float> %v1, <8 x float> poison, <8 x i32> zeroinitializer + %r = shufflevector <8 x float> %b0, <8 x float> %b1, <16 x i32> + ret <16 x float> %r +} + +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08: +; ALL: # %bb.0: +; ALL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %sv = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> + ret <16 x float> %sv +} + define <16 x i32> @shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i32> %a, <16 x i32> %b) { ; ALL-LABEL: shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00: ; ALL: # %bb.0: -- GitLab From ceabaa7e7a2d02b20cbd2b31e8336dedb1d4d9f5 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:48:03 +0000 Subject: [PATCH 148/933] [DAG] Fix some missing formatting when I rewrote the SUB(MAX,MIN) -> ABD patterns. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index db81f9199170..dcd0310734ad 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4024,13 +4024,13 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } // smax(a,b) - smin(a,b) --> abds(a,b) - if (hasOperation(ISD::ABDS, VT) && + if (hasOperation(ISD::ABDS, VT) && sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && sd_match(N1, m_SMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDS, DL, VT, A, B); // umax(a,b) - umin(a,b) --> abdu(a,b) - if (hasOperation(ISD::ABDU, VT) && + if (hasOperation(ISD::ABDU, VT) && sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && sd_match(N1, m_UMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDU, DL, VT, A, B); -- GitLab From c41286af3f30e099556c6edbef0001466afaefcb Mon Sep 17 00:00:00 2001 From: Pablo Antonio Martinez Date: Fri, 22 Mar 2024 11:53:29 +0000 Subject: [PATCH 149/933] [mlir][linalg] Emit a warning when tile_using_forall generates non thread-safe code (#80813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description** The documentation of `transform.structured.tile_using_forall` says: _"It is the user’s responsibility to ensure that num_threads/tile_sizes is a valid tiling specification (i.e. that only tiles parallel dimensions, e.g. in the Linalg case)."_ In other words, tiling a non-parallel dimension would generate code with data races which is not safe to parallelize. For example, consider this example (included in the tests in this PR): ``` func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { %0 = scf.forall (%arg2) in (8) shared_outs(%arg3 = %arg1) -> (tensor<300x8xf32>) { %1 = affine.min #map(%arg2) %2 = affine.max #map1(%1) %3 = affine.apply #map2(%arg2) %extracted_slice = tensor.extract_slice %arg0[%3, 0, 0] [%2, 300, 8] [1, 1, 1] : tensor<100x300x8xf32> to tensor %4 = linalg.generic {indexing_maps = [#map3, #map4], iterator_types = ["reduction", "parallel", "parallel"]} ins(%extracted_slice : tensor) outs(%arg3 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %5 = arith.addf %in, %out : f32 linalg.yield %5 : f32 } -> tensor<300x8xf32> scf.forall.in_parallel { tensor.parallel_insert_slice %4 into %arg3[0, 0] [300, 8] [1, 1] : tensor<300x8xf32> into tensor<300x8xf32> } } return %0 : tensor<300x8xf32> } ``` We can easily see that this is not safe to parallelize because all threads would be writing to the same position in `%arg3` (in the `scf.forall.in_parallel`. This PR detects wether it's safe to `tile_using_forall` and emits a warning in the case it is not. **Brief explanation** It first generates a vector of affine expressions representing the tile values and stores it in `dimExprs`. These affine expressions are compared with the affine expressions coming from the results of the affine map of each output in the linalg op. So going back to the previous example, the original transform is: ``` #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> #map1 = affine_map<(d0, d1, d2) -> (d1, d2)> func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { // expected-warning@+1 {{tiling is not thread safe at axis #0}} %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %1 = arith.addf %in, %out : f32 linalg.yield %1 : f32 } -> tensor<300x8xf32> return %0 : tensor<300x8xf32> } module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op) transform.yield } } ``` The `num_threads` attribute would be represented as `(d0)`. Because the linalg op has only one output (`arg1`) it would only check against the results of `#map1`, which are `(d1, d2)`. The idea is to check that all affine expressions in `dimExprs` are present in the output affine map. In this example, `d0` is not in `(d1, d2)`, so tiling that axis is considered not thread safe. --- .../Linalg/TransformOps/LinalgTransformOps.td | 4 +- mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp | 38 ++++- mlir/test/Dialect/Linalg/tile-to-forall.mlir | 141 ++++++++++++++++++ 3 files changed, 180 insertions(+), 3 deletions(-) diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index 4f34016066b4..c260fe3f7a46 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -1918,7 +1918,9 @@ def TileUsingForallOp : It is the user's responsibility to ensure that `num_threads/tile_sizes` is a valid tiling specification (i.e. that only tiles parallel dimensions, - e.g. in the Linalg case). + e.g. in the Linalg case). If the dimension is not parallelizable, a warning + is issued to notify the user that the generated code is not safe to + parallelize. If non-empty, the `mapping` is added as an attribute to the resulting `scf.forall`. diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp index 30aed850bed8..462f692615fa 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp @@ -304,6 +304,28 @@ static void calculateTileOffsetsAndSizes( } } +/// Returns a vector of bools representing if, for each axis, `op` can be tiled +/// without incurring in a race condition and thus it is thread-safe to do the +/// tiling. This is checked by iterating over numThreads and ensuring that the +/// corresponding iterator type is "parallel". If it is not, then we know that +/// such dimension is unsafe to tile. +SmallVector safeToTileToForall(mlir::MLIRContext *ctx, LinalgOp linalgOp, + ArrayRef numThreads) { + auto iterators = linalgOp.getIteratorTypesArray(); + SmallVector safeToTile(numThreads.size(), true); + + for (unsigned i = 0, e = numThreads.size(); i != e; i++) { + if (auto attr = llvm::dyn_cast_if_present(numThreads[i])) { + if (cast(attr).getValue().getSExtValue() > 1) { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } else { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } + return safeToTile; +} + /// Rewrite a TilingInterface `op` to a tiled `scf.forall`. The /// tiling is specified by the number of tiles/threads `numThreads` and the /// optional nominal tile size `nominalTileSizes`. If `nominalTilSizes` is @@ -314,8 +336,10 @@ static void calculateTileOffsetsAndSizes( /// size of data. /// It is the user's responsibility to ensure that `numThreads` is a valid /// tiling specification (i.e. that only tiles parallel dimensions, e.g. in the -/// Linalg case). If `omitTileOffsetBoundsCheck` is true, then the function will -/// assume that `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. +/// Linalg case). If the dimension is not parallelizable, a warning is issued to +/// notify the user that the generated code is not safe to parallelize. If +/// `omitTileOffsetBoundsCheck` is true, then the function will assume that +/// `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. static FailureOr tileToForallOpImpl( RewriterBase &b, TilingInterface op, ArrayRef numThreads, std::optional> nominalTileSizes, @@ -344,6 +368,16 @@ static FailureOr tileToForallOpImpl( return getValueOrCreateConstantIndexOp(b, loc, ofr); })); + LinalgOp linalgOp = dyn_cast(op.getOperation()); + if (linalgOp) { + // Check if tiling is thread safe and print a warning if not. + SmallVector tilingSafety = + safeToTileToForall(b.getContext(), linalgOp, numThreads); + for (size_t i = 0; i < tilingSafety.size(); i++) + if (!tilingSafety[i]) + op.emitWarning() << "tiling is not thread safe at axis #" << i; + } + // 1. Create the ForallOp. We don't use the lambda body-builder // version because we require the use of RewriterBase in the body, so we // manually move the insertion point to the body below. diff --git a/mlir/test/Dialect/Linalg/tile-to-forall.mlir b/mlir/test/Dialect/Linalg/tile-to-forall.mlir index abd807b3e4d3..12e2dea5530b 100644 --- a/mlir/test/Dialect/Linalg/tile-to-forall.mlir +++ b/mlir/test/Dialect/Linalg/tile-to-forall.mlir @@ -586,3 +586,144 @@ module attributes {transform.with_named_sequence} { transform.yield } } + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety1(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d1, d2)> + +func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #0}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<300x8xf32> + return %0 : tensor<300x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> + +func.func @tile_thread_safety3(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>) -> tensor<100x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<100x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100x8xf32> + return %0 : tensor<100x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2)> + +func.func @tile_thread_safety4(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>, %arg2 : tensor<8xf32>) -> (tensor<100x8xf32>, tensor<8xf32>) { + // expected-warning@+2 {{tiling is not thread safe at axis #0}} + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0:2 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["reduction", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1, %arg2 : tensor<100x8xf32>, tensor<8xf32>) { + ^bb0(%in: f32, %out1: f32, %out2: f32): + %1 = arith.addf %in, %out1 : f32 + %2 = arith.addf %in, %out2 : f32 + linalg.yield %1, %2 : f32, f32 + } -> (tensor<100x8xf32>, tensor<8xf32>) + return %0#0, %0#1 : tensor<100x8xf32>, tensor<8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety5(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 tile_sizes [10, 1] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +func.func @tile_thread_safety6(%A: tensor, %B: tensor, %C: tensor) -> tensor { + // expected-warning@below {{tiling is not thread safe at axis #2}} + %0 = linalg.matmul ins(%A, %B : tensor, tensor) + outs(%C : tensor) -> (tensor) + return %0 : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.matmul"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [2, 0, 8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} -- GitLab From d03f470cbdbae3f86469ea4d79bb54d3ef680512 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 06:54:33 -0500 Subject: [PATCH 150/933] [Clang] Make '-frtlib-add-rpath' include the standard library directory (#86217) Summary: The original intention of the `openmp-add-rpath` option was to add the rpath to the language runtime directory. However, the current implementation only adds it to the compiler's resource directory. This patch adds support for appending the `-rpath` to the compiler's standard library directory as well. Currently this is `/../lib/`. --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 447886531363..6b1fbba7abd0 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1142,7 +1142,11 @@ void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args, options::OPT_fno_rtlib_add_rpath, false)) return; - for (const auto &CandidateRPath : TC.getArchSpecificLibPaths()) { + SmallVector CandidateRPaths(TC.getArchSpecificLibPaths()); + if (const auto CandidateRPath = TC.getStdlibPath()) + CandidateRPaths.emplace_back(*CandidateRPath); + + for (const auto &CandidateRPath : CandidateRPaths) { if (TC.getVFS().exists(CandidateRPath)) { CmdArgs.push_back("-rpath"); CmdArgs.push_back(Args.MakeArgString(CandidateRPath)); -- GitLab From d51f1c442b1dc999267726a33e25b7d019726c89 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:14:26 -0400 Subject: [PATCH 151/933] [DirectX][Docs] Add DXILIntrinsicExpansion Pass to DXILArchitecture.rst (#86198) Completes #84839 --------- Co-authored-by: Farzon Lotfi --- llvm/docs/DirectX/DXILArchitecture.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/docs/DirectX/DXILArchitecture.rst b/llvm/docs/DirectX/DXILArchitecture.rst index d6712bea4f77..32b1e72deae7 100644 --- a/llvm/docs/DirectX/DXILArchitecture.rst +++ b/llvm/docs/DirectX/DXILArchitecture.rst @@ -61,6 +61,19 @@ on the utilities described in "Common Code" above in order to share logic with both the DirectX backend and with Clang's codegen of HLSL support as much as possible. +The DirectX Intrinsic Expansion Pass +==================================== +There are intrinsics that don't map directly to DXIL Ops. In some cases +an intrinsic needs to be expanded to a set of LLVM IR instructions. In +other cases an intrinsic needs modifications to the arguments or return +values of a DXIL Op. The `DXILIntrinsicExpansion` pass handles all +the cases where our intrinsics don't have a one to one mapping. This +pass may also be used when the expansion is specific to DXIL to keep +implementation details out of CodeGen. Finally, there is an expectation +that we maintain vector types through this pass. Therefore, best +practice would be to avoid scalarization in this pass. + + The DirectX Backend =================== -- GitLab From fe64b26df9429c82f706424dcdae3d65723c3e5e Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 12:20:52 +0000 Subject: [PATCH 152/933] NFC Rename LoadBitcodeIntoNewDbgInforFormat to LoadBitcodeIntoNewDbgInfoFormat (drop additional 'r' before Format) --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 3fc8141381c6..6ee93f17792b 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -104,7 +104,7 @@ static cl::opt ExpandConstantExprs( /// of debug intrinsics). UNSET is treated as FALSE, so the default action /// is to do nothing. Individual tools can override this to incrementally add /// support for the RemoveDIs format. -cl::opt LoadBitcodeIntoNewDbgInforFormat( +cl::opt LoadBitcodeIntoNewDbgInfoFormat( "load-bitcode-into-experimental-debuginfo-iterators", cl::Hidden, cl::desc("Load bitcode directly into the new debug info format (regardless " "of input format)")); @@ -4300,11 +4300,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata, ParserCallbacks Callbacks) { - // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInforFormat + // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInfoFormat // has been set to true (default action: load into the old debug format). TheModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat && - LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_TRUE; + LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_TRUE; this->ValueTypeCallback = std::move(Callbacks.ValueType); if (ResumeBit) { -- GitLab From e1f50fdc03efecb5da39c1df4fc08d2ce5da90e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:35:36 +0100 Subject: [PATCH 153/933] [mlir] Remove unused and untested `shouldSplitInputFile`. (#85622) This was changed by #84765 but turned out to be buggy. Since it isn't used and isn't tested, it is probably best to remove it. --- mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h | 1 - 1 file changed, 1 deletion(-) diff --git a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h index 8adc80908de1..4f7f83cdb473 100644 --- a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h +++ b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h @@ -144,7 +144,6 @@ public: splitInputFileFlag = std::move(splitMarker); return *this; } - bool shouldSplitInputFile() const { return splitInputFileFlag.empty(); } StringRef inputSplitMarker() const { return splitInputFileFlag; } /// Set whether to merge the output chunks into one file using the given -- GitLab From 83da7b6338053ca04cf0afe3c70ef5b8a9f6d300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:36:07 +0100 Subject: [PATCH 154/933] [mlir] Extend split marker tests of `mlir-opt` and `mlir-pdll`. (#85620) Recently #84765 made the split markers of various tools configurable but did not test *not* using the split markers for two of them. This PR adds those tests. --- mlir/test/mlir-opt/split-markers.mlir | 32 ++++++++++++++++---------- mlir/test/mlir-pdll/split-markers.pdll | 12 ++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/mlir/test/mlir-opt/split-markers.mlir b/mlir/test/mlir-opt/split-markers.mlir index 665a37f31770..f372654bcc8d 100644 --- a/mlir/test/mlir-opt/split-markers.mlir +++ b/mlir/test/mlir-opt/split-markers.mlir @@ -1,12 +1,17 @@ // Check near-miss mechanics: // RUN: mlir-opt --split-input-file --verify-diagnostics %s 2> %t \ -// RUN: && FileCheck --input-file %t %s +// RUN: && FileCheck --input-file %t --check-prefix=CHECK-DEFAULT %s // RUN: cat %t // Check that (1) custom input splitter and (2) custom output splitters work. -// RUN: mlir-opt %s -split-input-file="// CHECK: ""----" \ +// RUN: mlir-opt %s -split-input-file="// CHECK-DEFAULT: ""----" \ // RUN: -output-split-marker="// ---- next split ----" \ -// RUN: | FileCheck --check-prefix=CHECK-SPLITTERS %s +// RUN: | FileCheck --check-prefix=CHECK-CUSTOM %s + +// Check that (3) the input is not split if `-split-input-file` is not given. +// RUN: mlir-opt %s 2> %t \ +// RUN: || FileCheck --input-file %t --check-prefix=CHECK-NOSPLIT %s +// RUN: cat %t func.func @main() {return} @@ -14,22 +19,25 @@ func.func @main() {return} // expected-note @+1 {{see existing symbol definition here}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- +// CHECK-NOSPLIT: error: redefinition of symbol named 'main' +func.func @main() {return} + // expected-error @+1 {{redefinition of symbol named 'foo'}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- func.func @bar2() {return } // No error flagged at the end for a near miss. // ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module diff --git a/mlir/test/mlir-pdll/split-markers.pdll b/mlir/test/mlir-pdll/split-markers.pdll index 45e409a83836..2b314538004f 100644 --- a/mlir/test/mlir-pdll/split-markers.pdll +++ b/mlir/test/mlir-pdll/split-markers.pdll @@ -9,6 +9,10 @@ // RUN: -split-input-file="// ""=====" -output-split-marker "// #####" \ // RUN: | FileCheck -check-prefix=CHECK-CUSTOM %s +// Check that (5) the input is not split if `-split-input-file` is not given. +// RUN: mlir-pdll %s \ +// RUN: | FileCheck -check-prefix=CHECK-NOSPLIT %s + // CHECK-DEFAULT: Module // CHECK-DEFAULT-NEXT: PatternDecl // CHECK-DEFAULT-NOT: PatternDecl @@ -25,6 +29,14 @@ // CHECK-CUSTOM-NEXT: PatternDecl // CHECK-CUSTOM-NOT: PatternDecl +// CHECK-NOSPLIT: Module +// CHECK-NOSPLIT-NEXT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module + Pattern => erase op; // ----- -- GitLab From 04a6e0f1634f9a53120c27a30250d26dff4ada1c Mon Sep 17 00:00:00 2001 From: Paul Robinson Date: Fri, 22 Mar 2024 05:35:09 -0700 Subject: [PATCH 155/933] [X86][Headers] change 'yields' to 'returns' in more places --- clang/lib/Headers/mmintrin.h | 12 +++++----- clang/lib/Headers/smmintrin.h | 4 ++-- clang/lib/Headers/xmmintrin.h | 44 +++++++++++++++++------------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/Headers/mmintrin.h b/clang/lib/Headers/mmintrin.h index 962d24738e7a..4e154e2d8593 100644 --- a/clang/lib/Headers/mmintrin.h +++ b/clang/lib/Headers/mmintrin.h @@ -1141,7 +1141,7 @@ _mm_xor_si64(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1163,7 +1163,7 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1185,7 +1185,7 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -1207,7 +1207,7 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1229,7 +1229,7 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1251,7 +1251,7 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/smmintrin.h b/clang/lib/Headers/smmintrin.h index 9fb9cc9b0134..b3fec474e35a 100644 --- a/clang/lib/Headers/smmintrin.h +++ b/clang/lib/Headers/smmintrin.h @@ -1188,7 +1188,7 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M, /// Compares each of the corresponding 64-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// @@ -2303,7 +2303,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) { /// integer vectors to determine if the values in the first operand are /// greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h index 040194786a27..1ef89de9c9f5 100644 --- a/clang/lib/Headers/xmmintrin.h +++ b/clang/lib/Headers/xmmintrin.h @@ -484,7 +484,7 @@ _mm_xor_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both /// operands for equality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -509,7 +509,7 @@ _mm_cmpeq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -531,7 +531,7 @@ _mm_cmpeq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than the /// corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -557,7 +557,7 @@ _mm_cmplt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -579,7 +579,7 @@ _mm_cmplt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than or /// equal to the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in /// the low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -605,7 +605,7 @@ _mm_cmple_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -627,7 +627,7 @@ _mm_cmple_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -655,7 +655,7 @@ _mm_cmpgt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -677,7 +677,7 @@ _mm_cmpgt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -705,7 +705,7 @@ _mm_cmpge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -726,7 +726,7 @@ _mm_cmpge_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both operands /// for inequality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -752,7 +752,7 @@ _mm_cmpneq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for inequality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -775,7 +775,7 @@ _mm_cmpneq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -802,7 +802,7 @@ _mm_cmpnlt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -825,7 +825,7 @@ _mm_cmpnlt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -852,7 +852,7 @@ _mm_cmpnle_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -875,7 +875,7 @@ _mm_cmpnle_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -904,7 +904,7 @@ _mm_cmpngt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -927,7 +927,7 @@ _mm_cmpngt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -956,7 +956,7 @@ _mm_cmpnge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -3061,7 +3061,7 @@ _mm_movemask_ps(__m128 __a) /// [4 x float], using the operation specified by the immediate integer /// operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// @@ -3096,7 +3096,7 @@ _mm_movemask_ps(__m128 __a) /// vectors of [4 x float], using the operation specified by the immediate /// integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// -- GitLab From 31a9a4b83720af79110941824abe28fc6ff42355 Mon Sep 17 00:00:00 2001 From: yronglin Date: Fri, 22 Mar 2024 20:45:17 +0800 Subject: [PATCH 156/933] [libc++] Implement LWG3528 (`make_from_tuple` can perform (the equivalent of) a C-style cast) (#85263) Implement [LWG3528](https://wg21.link/LWG3528). Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, the standard allows to impose requirements, we constraint `std::make_from_tuple` to make `std::make_from_tuple` SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of `std::__make_from_tuple_impl` so that `std::__make_from_tuple_impl` will have the same advantages when used alone. --------- Signed-off-by: yronglin --- libcxx/docs/Status/Cxx23Issues.csv | 2 +- libcxx/include/tuple | 28 +++++- .../tuple.apply/make_from_tuple.pass.cpp | 88 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv index 8de265f4d1b6..ebdc4a745c9f 100644 --- a/libcxx/docs/Status/Cxx23Issues.csv +++ b/libcxx/docs/Status/Cxx23Issues.csv @@ -77,7 +77,7 @@ `3523 `__,"``iota_view::sentinel`` is not always ``iota_view``'s sentinel","June 2021","","","|ranges|" `3526 `__,"Return types of ``uses_allocator_construction_args`` unspecified","June 2021","","" `3527 `__,"``uses_allocator_construction_args`` handles rvalue pairs of rvalue references incorrectly","June 2021","","" -`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","","" +`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","|Complete|","19.0" `3529 `__,"``priority_queue(first, last)`` should construct ``c`` with ``(first, last)``","June 2021","|Complete|","14.0" `3530 `__,"``BUILTIN-PTR-MEOW`` should not opt the type out of syntactic checks","June 2021","","" `3532 `__,"``split_view::inner-iterator::operator++(int)`` should depend on ``Base``","June 2021","","","|ranges|" diff --git a/libcxx/include/tuple b/libcxx/include/tuple index f78db061b844..a9f0d680fe0e 100644 --- a/libcxx/include/tuple +++ b/libcxx/include/tuple @@ -1377,15 +1377,41 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) apply(_Fn&& __f, _Tuple&& std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) +#if _LIBCPP_STD_VER >= 20 template inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>) + noexcept(noexcept(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))) + requires is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...> { + return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...); +} +#else +template +inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>, + enable_if_t(std::forward<_Tuple>(__t)))...>> * = nullptr) _LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...)) +#endif // _LIBCPP_STD_VER >= 20 + +template >>::type, class = void> +inline constexpr bool __can_make_from_tuple = false; +template +inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, __tuple_indices<_Idx...>, + enable_if_t(std::declval<_Tuple>()))...>>> = true; + +// Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, +// the standard allows to impose requirements, we constraint std::make_from_tuple to make std::make_from_tuple +// SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of std::__make_from_tuple_impl +// so that std::__make_from_tuple_impl will have the same advantages when used alone. +#if _LIBCPP_STD_VER >= 20 template + requires __can_make_from_tuple<_Tp, _Tuple> // strengthen +#else +template >> // strengthen +#endif // _LIBCPP_STD_VER >= 20 inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t) _LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>( std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) - # undef _LIBCPP_NOEXCEPT_RETURN # endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp index e3a21149c21e..d7374351afa8 100644 --- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp +++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp @@ -195,6 +195,94 @@ void test_noexcept() { } } +namespace LWG3528 { +template +auto test_make_from_tuple(T&&, Tuple&& t) -> decltype(std::make_from_tuple(t), uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +template +auto test_make_from_tuple_impl(T&&, Tuple&& t) + -> decltype(std::__make_from_tuple_impl( + t, typename std::__make_tuple_indices< std::tuple_size_v>>::type{}), + uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple_impl(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple_impl = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +struct A { + int a; +}; +struct B : public A {}; + +struct C { + C(const B&) {} +}; + +enum class D { + ONE, + TWO, +}; + +// Test std::make_from_tuple constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// const_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// static_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// Test std::__make_from_tuple_impl constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// const_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// static_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +} // namespace LWG3528 + int main(int, char**) { test_constexpr_construction(); -- GitLab From 3b3de48fd84b8269d5f45ee0a9dc6b7448368424 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 06:07:17 -0700 Subject: [PATCH 157/933] [BOLT] Add BB index to BAT (#86044) --- bolt/docs/BAT.md | 11 ++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 - lld/MachO/Driver.cpp | 42 +++++++++++++++++-- lld/MachO/InputSection.cpp | 38 ----------------- lld/MachO/InputSection.h | 3 -- lld/MachO/ObjC.cpp | 16 ++++--- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 92 insertions(+), 77 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 86a287db72a4..bc9cc8ce6cf5 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { - // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { + // NVPTX/AMDGPU does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index 5618b2cba72e..f23cb71ebfb0 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,5 +2,4 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all -// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 919a14b8bcf0..36248925d65a 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - addInputSection(isec); + inputSections.push_back(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,18 +1220,53 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); + int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - for (const Subsection &subsection : section->subsections) - addInputSection(subsection.isec); + ConcatOutputSection *osec = nullptr; + for (const Subsection &subsection : section->subsections) { + if (auto *isec = dyn_cast(subsection.isec)) { + if (isec->isCoalescedWeak()) + continue; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + continue; + } + isec->outSecOff = inputOrder++; + if (!osec) + osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + } } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } + assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1387,7 +1422,6 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); - inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 22930d52dd1d..8f5affb1dc21 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,44 +37,6 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; -int macho::inputSectionsOrder = 0; - -// Call this function to add a new InputSection and have it routed to the -// appropriate container. Depending on its type and current config, it will -// either be added to 'inputSections' vector or to a synthetic section. -void lld::macho::addInputSection(InputSection *inputSection) { - if (auto *isec = dyn_cast(inputSection)) { - if (isec->isCoalescedWeak()) - return; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - return; - } - isec->outSecOff = inputSectionsOrder++; - auto *osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = dyn_cast(inputSection)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputSectionsOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputSectionsOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = dyn_cast(inputSection)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputSectionsOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - - assert(inputSectionsOrder <= UnspecifiedInputOrder); -} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index 694bdf734907..b25f0638f4c6 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,8 +302,6 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; -// This is used as a counter for specyfing input order for input sections -extern int inputSectionsOrder; namespace section_names { @@ -371,7 +369,6 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names -void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 5902b82d30f5..40df2243b26f 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - addInputSection(newCatList); + allInputSections.push_back(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - addInputSection(newBodySec); + allInputSections.push_back(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,11 +1221,9 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - uint32_t bufSize = len + 1; - auto &data = newSectionData(bufSize); + auto &data = newSectionData(len + 1); char *strData = reinterpret_cast(data.data()); - // Copy the string chars and null-terminator - memcpy(strData, str, bufSize); + strncpy(strData, str, len); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 1b3694528de1..7ee3261ce307 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - addInputSection(in.imageLoaderCache); + inputSections.push_back(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - addInputSection(objcSelref); + inputSections.push_back(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b3f98dff75469b115e3d4b1f10cbf270c8ee81af Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 13:52:11 +0000 Subject: [PATCH 158/933] [RemoveDIs] Load into new debug info format by default in llvm-lto and llvm-lto2 (#86271) Directly load all bitcode into the new debug info format in `llvm-lto` and `llvm-lto2`. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/lib/LTO/LTO.cpp | 4 +++- llvm/tools/llvm-lto/llvm-lto.cpp | 4 ++++ llvm/tools/llvm-lto2/llvm-lto2.cpp | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index b58418c64a11..53060df7f503 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -579,7 +579,9 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel, const Config &Conf) : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel), Ctx(Conf), CombinedModule(std::make_unique("ld-temp.o", Ctx)), - Mover(std::make_unique(*CombinedModule)) {} + Mover(std::make_unique(*CombinedModule)) { + CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat; +} LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) { diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp index 7943d6952b82..3c452b650cee 100644 --- a/llvm/tools/llvm-lto/llvm-lto.cpp +++ b/llvm/tools/llvm-lto/llvm-lto.cpp @@ -270,6 +270,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -943,6 +944,9 @@ int main(int argc, char **argv) { InitLLVM X(argc, argv); cl::HideUnrelatedOptions({<OCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index d5de4f6b1a27..f222d02bd7ce 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -193,6 +193,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; static void check(Error E, std::string Msg) { if (!E) @@ -228,6 +229,9 @@ static int usage() { static int run(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. -- GitLab From a277dd82d89a17849cf99fb335660ea0b8894878 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 13:43:58 +0000 Subject: [PATCH 159/933] [X86] vector-half-conversions.ll - add v4f16->v4i32 fptosi/fptoui test coverage --- .../CodeGen/X86/vector-half-conversions.ll | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll index ba21af231985..563cf0165013 100644 --- a/llvm/test/CodeGen/X86/vector-half-conversions.ll +++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll @@ -4989,3 +4989,257 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind { %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> ret <4 x i32> %ext } + +define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX-LABEL: fptosi_4f16_to_4i32: +; AVX: # %bb.0: +; AVX-NEXT: subq $72, %rsp +; AVX-NEXT: vmovdqa %xmm0, %xmm1 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX-NEXT: addq $72, %rsp +; AVX-NEXT: retq +; +; F16C-LABEL: fptosi_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512-LABEL: fptosi_4f16_to_4i32: +; AVX512: # %bb.0: +; AVX512-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-NEXT: vcvttps2dq %ymm0, %ymm0 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq + %cvt = fptosi <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} + +define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind { +; AVX1-LABEL: fptoui_2f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $40, %rsp +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: addq $40, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_2f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $40, %rsp +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX2-NEXT: addq $40, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_2f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vpsrld $16, %xmm0, %xmm1 +; F16C-NEXT: vcvtph2ps %xmm1, %xmm1 +; F16C-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 +; F16C-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; F16C-NEXT: vcvttps2dq %xmm0, %xmm1 +; F16C-NEXT: vpsrad $31, %xmm1, %xmm2 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; F16C-NEXT: vcvttps2dq %xmm0, %xmm0 +; F16C-NEXT: vpand %xmm2, %xmm0, %xmm0 +; F16C-NEXT: vpor %xmm0, %xmm1, %xmm0 +; F16C-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_2f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512F-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512F-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_2f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512-FASTLANE-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512-FASTLANE-NEXT: vcvttps2udq %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <2 x half> %a to <2 x i32> + %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> + ret <4 x i32> %ext +} + +define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX1-LABEL: fptoui_4f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $72, %rsp +; AVX1-NEXT: vmovdqa %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX1-NEXT: addq $72, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_4f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $72, %rsp +; AVX2-NEXT: vmovdqa %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX2-NEXT: addq $72, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm1 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: vorps %ymm0, %ymm1, %ymm0 +; F16C-NEXT: vblendvps %ymm1, %ymm0, %ymm1, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_4f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_4f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-FASTLANE-NEXT: vcvttps2udq %ymm0, %ymm0 +; AVX512-FASTLANE-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-FASTLANE-NEXT: vzeroupper +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} -- GitLab From f82d0187a7e581d4f8f825021dbcb08e8eb37d61 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 14:00:21 +0000 Subject: [PATCH 160/933] [AArch64] Add a test to show incorrect latencies into Bundle instructions. NFC --- llvm/test/CodeGen/AArch64/misched-bundle.mir | 195 +++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/misched-bundle.mir diff --git a/llvm/test/CodeGen/AArch64/misched-bundle.mir b/llvm/test/CodeGen/AArch64/misched-bundle.mir new file mode 100644 index 000000000000..a947c04a4229 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/misched-bundle.mir @@ -0,0 +1,195 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a510 -run-pass=machine-scheduler -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s +# REQUIRES: asserts + +# CHECK: SU(0): renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z0 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(1): renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(2): renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(3): renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(4): renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(5): renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(6): $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0(tied-def 0), killed renamable $z1, killed renamable $z2 +# CHECK-NEXT: # preds left : 4 +# CHECK-NEXT: # succs left : 2 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 4 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 4 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(2): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(1): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(0): Out Latency=1 +# CHECK-NEXT: SU(0): Data Latency=3 Reg=$z0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(8): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(7): Anti Latency=0 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(7): BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 +# CHECK-NEXT: # preds left : 5 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Anti Latency=0 +# CHECK-NEXT: SU(5): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(4): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(3): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(1): Out Latency=1 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Data Latency=0 Reg=$z1 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(8): ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 7 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(9): ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 8 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z1 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: ExitSU: RET_ReallyLR +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 0 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 + +--- +name: test +alignment: 4 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + + ; CHECK-LABEL: name: test + ; CHECK: liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, renamable $z1, killed renamable $z2 + ; CHECK-NEXT: renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit $z4, implicit $z3 { + ; CHECK-NEXT: $z1 = MOVPRFX_ZZ $z5 + ; CHECK-NEXT: $z1 = FMLA_ZPmZZ_H renamable $p0, internal $z1, renamable $z4, renamable $z3 + ; CHECK-NEXT: } + ; CHECK-NEXT: ST1H renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: RET_ReallyLR + + renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size) + renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size) + renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size) + renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size) + renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size) + renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size) + $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, killed renamable $z1, killed renamable $z2 + BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 { + $z1 = MOVPRFX_ZZ $z5 + $z1 = FMLA_ZPmZZ_H renamable $p0, internal killed $z1, killed renamable $z4, killed renamable $z3 + } + ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size) + ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size) + RET_ReallyLR + +... -- GitLab From e54af608160350baa7ae1b8069f916eb625beadd Mon Sep 17 00:00:00 2001 From: Hirofumi Nakamura Date: Fri, 22 Mar 2024 23:11:36 +0900 Subject: [PATCH 161/933] [clang-format] Added AlignConsecutiveTableGenBreakingDAGArgColons option. (#86150) The option to specify the style of alignment of the colons inside TableGen's DAGArg. --- clang/docs/ClangFormatStyleOptions.rst | 145 ++++++++++++++++++ clang/include/clang/Format/Format.h | 17 ++ clang/lib/Format/Format.cpp | 3 + clang/lib/Format/FormatToken.h | 1 + clang/lib/Format/TokenAnnotator.cpp | 18 ++- clang/lib/Format/WhitespaceManager.cpp | 6 + clang/lib/Format/WhitespaceManager.h | 3 + clang/unittests/Format/FormatTestTableGen.cpp | 32 ++++ clang/unittests/Format/TokenAnnotatorTest.cpp | 16 ++ 9 files changed, 236 insertions(+), 5 deletions(-) diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index be021dfc5c08..2ee36f24d7ce 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -955,6 +955,151 @@ the configuration (without a prefix: ``Auto``). } +.. _AlignConsecutiveTableGenBreakingDAGArgColons: + +**AlignConsecutiveTableGenBreakingDAGArgColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` + Style of aligning consecutive TableGen DAGArg operator colons. + If enabled, align the colon inside DAGArg which have line break inside. + This works only when TableGenBreakInsideDAGArg is BreakElements or + BreakAll and the DAGArg is not excepted by + TableGenBreakingDAGArgOperators's effect. + + .. code-block:: c++ + + let dagarg = (ins + a :$src1, + aa :$src2, + aaa:$src3 + ) + + Nested configuration flags: + + Alignment options. + + They can also be read as a whole for compatibility. The choices are: + - None + - Consecutive + - AcrossEmptyLines + - AcrossComments + - AcrossEmptyLinesAndComments + + For example, to align across empty lines and not across comments, either + of these work. + + .. code-block:: c++ + + AlignConsecutiveTableGenBreakingDAGArgColons: AcrossEmptyLines + + AlignConsecutiveTableGenBreakingDAGArgColons: + Enabled: true + AcrossEmptyLines: true + AcrossComments: false + + * ``bool Enabled`` Whether aligning is enabled. + + .. code-block:: c++ + + #define SHORT_NAME 42 + #define LONGER_NAME 0x007f + #define EVEN_LONGER_NAME (2) + #define foo(x) (x * x) + #define bar(y, z) (y + z) + + int a = 1; + int somelongname = 2; + double c = 3; + + int aaaa : 1; + int b : 12; + int ccc : 8; + + int aaaa = 12; + float b = 23; + std::string ccc; + + * ``bool AcrossEmptyLines`` Whether to align across empty lines. + + .. code-block:: c++ + + true: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + false: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + * ``bool AcrossComments`` Whether to align across comments. + + .. code-block:: c++ + + true: + int d = 3; + /* A comment. */ + double e = 4; + + false: + int d = 3; + /* A comment. */ + double e = 4; + + * ``bool AlignCompound`` Only for ``AlignConsecutiveAssignments``. Whether compound assignments + like ``+=`` are aligned along with ``=``. + + .. code-block:: c++ + + true: + a &= 2; + bbb = 2; + + false: + a &= 2; + bbb = 2; + + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment + operators are left-padded to the same length as long ones in order to + put all assignment operators to the right of the left hand side. + + .. code-block:: c++ + + true: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + false: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + .. _AlignConsecutiveTableGenCondOperatorColons: **AlignConsecutiveTableGenCondOperatorColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index 7ad2579bf777..0720c8283cd7 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -414,6 +414,21 @@ struct FormatStyle { /// \version 17 ShortCaseStatementsAlignmentStyle AlignConsecutiveShortCaseStatements; + /// Style of aligning consecutive TableGen DAGArg operator colons. + /// If enabled, align the colon inside DAGArg which have line break inside. + /// This works only when TableGenBreakInsideDAGArg is BreakElements or + /// BreakAll and the DAGArg is not excepted by + /// TableGenBreakingDAGArgOperators's effect. + /// \code + /// let dagarg = (ins + /// a :$src1, + /// aa :$src2, + /// aaa:$src3 + /// ) + /// \endcode + /// \version 19 + AlignConsecutiveStyle AlignConsecutiveTableGenBreakingDAGArgColons; + /// Style of aligning consecutive TableGen cond operator colons. /// Align the colons of cases inside !cond operators. /// \code @@ -4879,6 +4894,8 @@ struct FormatStyle { AlignConsecutiveMacros == R.AlignConsecutiveMacros && AlignConsecutiveShortCaseStatements == R.AlignConsecutiveShortCaseStatements && + AlignConsecutiveTableGenBreakingDAGArgColons == + R.AlignConsecutiveTableGenBreakingDAGArgColons && AlignConsecutiveTableGenCondOperatorColons == R.AlignConsecutiveTableGenCondOperatorColons && AlignConsecutiveTableGenDefinitionColons == diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 63ec3a88978d..46ed5baaeace 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -895,6 +895,8 @@ template <> struct MappingTraits { IO.mapOptional("AlignConsecutiveMacros", Style.AlignConsecutiveMacros); IO.mapOptional("AlignConsecutiveShortCaseStatements", Style.AlignConsecutiveShortCaseStatements); + IO.mapOptional("AlignConsecutiveTableGenBreakingDAGArgColons", + Style.AlignConsecutiveTableGenBreakingDAGArgColons); IO.mapOptional("AlignConsecutiveTableGenCondOperatorColons", Style.AlignConsecutiveTableGenCondOperatorColons); IO.mapOptional("AlignConsecutiveTableGenDefinitionColons", @@ -1408,6 +1410,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) { LLVMStyle.AlignConsecutiveDeclarations = {}; LLVMStyle.AlignConsecutiveMacros = {}; LLVMStyle.AlignConsecutiveShortCaseStatements = {}; + LLVMStyle.AlignConsecutiveTableGenBreakingDAGArgColons = {}; LLVMStyle.AlignConsecutiveTableGenCondOperatorColons = {}; LLVMStyle.AlignConsecutiveTableGenDefinitionColons = {}; LLVMStyle.AlignEscapedNewlines = FormatStyle::ENAS_Right; diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 06f567059c35..2ddcd5259446 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -152,6 +152,7 @@ namespace format { TYPE(TableGenCondOperatorComma) \ TYPE(TableGenDAGArgCloser) \ TYPE(TableGenDAGArgListColon) \ + TYPE(TableGenDAGArgListColonToAlign) \ TYPE(TableGenDAGArgListComma) \ TYPE(TableGenDAGArgListCommaToBreak) \ TYPE(TableGenDAGArgOpener) \ diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 94d2266555f6..a5cafcbacaa5 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -975,12 +975,15 @@ private: // DagArg ::= Value [":" TokVarName] | TokVarName // Appears as a part of SimpleValue6. - bool parseTableGenDAGArg() { + bool parseTableGenDAGArg(bool AlignColon = false) { if (tryToParseTableGenTokVar()) return true; if (parseTableGenValue()) { if (CurrentToken && CurrentToken->is(tok::colon)) { - CurrentToken->setType(TT_TableGenDAGArgListColon); + if (AlignColon) + CurrentToken->setType(TT_TableGenDAGArgListColonToAlign); + else + CurrentToken->setType(TT_TableGenDAGArgListColon); skipToNextNonComment(); return tryToParseTableGenTokVar(); } @@ -1051,8 +1054,11 @@ private: skipToNextNonComment(); return true; } - if (!parseTableGenDAGArg()) + if (!parseTableGenDAGArg( + BreakInside && + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled)) { return false; + } FirstDAGArgListElm = false; } return false; @@ -5130,8 +5136,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Left.is(tok::r_brace) && Right.is(tok::r_square)) return true; // Do not insert around colon in DAGArg and cond operator. - if (Right.is(TT_TableGenDAGArgListColon) || - Left.is(TT_TableGenDAGArgListColon)) { + if (Right.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign) || + Left.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign)) { return false; } if (Right.is(TT_TableGenCondOperatorColon)) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 753be25bfd67..fef85abf79a3 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -112,6 +112,7 @@ const tooling::Replacements &WhitespaceManager::generateReplacements() { alignConsecutiveBitFields(); alignConsecutiveAssignments(); if (Style.isTableGen()) { + alignConsecutiveTableGenBreakingDAGArgColons(); alignConsecutiveTableGenCondOperatorColons(); alignConsecutiveTableGenDefinitions(); } @@ -981,6 +982,11 @@ void WhitespaceManager::alignConsecutiveShortCaseStatements() { Changes); } +void WhitespaceManager::alignConsecutiveTableGenBreakingDAGArgColons() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenBreakingDAGArgColons, + TT_TableGenDAGArgListColonToAlign); +} + void WhitespaceManager::alignConsecutiveTableGenCondOperatorColons() { alignConsecutiveColons(Style.AlignConsecutiveTableGenCondOperatorColons, TT_TableGenCondOperatorColon); diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h index 0ebc6cf8377c..98cf4a260cc4 100644 --- a/clang/lib/Format/WhitespaceManager.h +++ b/clang/lib/Format/WhitespaceManager.h @@ -235,6 +235,9 @@ private: /// Align consecutive short case statements over all \c Changes. void alignConsecutiveShortCaseStatements(); + /// Align consecutive TableGen DAGArg colon over all \c Changes. + void alignConsecutiveTableGenBreakingDAGArgColons(); + /// Align consecutive TableGen cond operator colon over all \c Changes. void alignConsecutiveTableGenCondOperatorColons(); diff --git a/clang/unittests/Format/FormatTestTableGen.cpp b/clang/unittests/Format/FormatTestTableGen.cpp index c96866f0840f..8ca6bf97e5a6 100644 --- a/clang/unittests/Format/FormatTestTableGen.cpp +++ b/clang/unittests/Format/FormatTestTableGen.cpp @@ -411,6 +411,38 @@ TEST_F(FormatTestTableGen, DAGArgBreakAll) { Style); } +TEST_F(FormatTestTableGen, DAGArgAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll; + Style.TableGenBreakingDAGArgOperators = {"ins", "outs"}; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a:$src1,\n" + " aa:$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a :$src1,\n" + " aa :$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); +} + TEST_F(FormatTestTableGen, CondOperatorAlignment) { FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); Style.ColumnLimit = 60; diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 1aa855b34198..2539d3d76ef0 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2424,6 +2424,22 @@ TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) { EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma); EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser); + + // If TableGenBreakingDAGArgOperators is enabled, it uses + // TT_TableGenDAGArgListColonToAlign to annotate the colon to align. + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, + TT_TableGenDAGArgOperatorToBreak); // ins + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign); + + Tokens = AnnotateValue("(other type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon); } TEST_F(TokenAnnotatorTest, UnderstandConstructors) { -- GitLab From 46b853a82ce64e5213f8dfa2c12c6e6a950018a0 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Fri, 22 Mar 2024 14:17:06 +0000 Subject: [PATCH 162/933] [MC][COFF][AArch64] Treat ARM64EC/X as ARM64 for relocations (#86019) Since ARM64EC/X objects use regular ARM64 relocations, any special handling must be done for them too. --- llvm/lib/MC/WinCOFFObjectWriter.cpp | 4 ++-- llvm/test/MC/AArch64/coff-relocations.s | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index f265fafa59e7..3c9ff71b6b06 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -266,7 +266,7 @@ WinCOFFWriter::WinCOFFWriter(WinCOFFObjectWriter &OWriter, // limited range for the immediate offset (+/- 1 MB); create extra offset // label symbols with regular intervals to allow referencing a // non-temporary symbol that is close enough. - UseOffsetLabels = Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64; + UseOffsetLabels = COFF::isAnyArm64(Header.Machine); } COFFSymbol *WinCOFFWriter::createSymbol(StringRef Name) { @@ -954,7 +954,7 @@ void WinCOFFWriter::recordRelocation(MCAssembler &Asm, Reloc.Data.Type == COFF::IMAGE_REL_I386_REL32) || (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARMNT && Reloc.Data.Type == COFF::IMAGE_REL_ARM_REL32) || - (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64 && + (COFF::isAnyArm64(Header.Machine) && Reloc.Data.Type == COFF::IMAGE_REL_ARM64_REL32)) FixedValue += 4; diff --git a/llvm/test/MC/AArch64/coff-relocations.s b/llvm/test/MC/AArch64/coff-relocations.s index fb67a21992c6..2370fd9fb436 100644 --- a/llvm/test/MC/AArch64/coff-relocations.s +++ b/llvm/test/MC/AArch64/coff-relocations.s @@ -1,7 +1,11 @@ // RUN: llvm-mc -triple aarch64-windows -filetype obj -o %t.obj %s -// RUN: llvm-readobj -r %t.obj | FileCheck %s +// RUN: llvm-mc -triple arm64ec-windows -filetype obj -o %t-ec.obj %s +// RUN: llvm-readobj -r %t.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64 +// RUN: llvm-readobj -r %t-ec.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64EC // RUN: llvm-objdump --no-print-imm-hex -d %t.obj | FileCheck %s --check-prefix=DISASM +// RUN: llvm-objdump --no-print-imm-hex -d %t-ec.obj | FileCheck %s --check-prefix=DISASM // RUN: llvm-objdump -s %t.obj | FileCheck %s --check-prefix=DATA +// RUN: llvm-objdump -s %t-ec.obj | FileCheck %s --check-prefix=DATA // IMAGE_REL_ARM64_ADDR32 .Linfo_foo: @@ -71,8 +75,10 @@ tbz x0, #0, target // IMAGE_REL_ARM64_REL32 because IMAGE_REL_ARM64_REL64 does not exist. .xword .Linfo_foo - .Ltable -// CHECK: Format: COFF-ARM64 -// CHECK: Arch: aarch64 +// CHECK-ARM64: Format: COFF-ARM64 +// CHECK-ARM64EC: Format: COFF-ARM64EC +// CHECK-ARM64: Arch: aarch64 +// CHECK-ARM64EC: Arch: aarch64 // CHECK: AddressSize: 64bit // CHECK: Relocations [ // CHECK: Section (1) .text { -- GitLab From d231e3b10ead90e4360f7ceb88e4bca9d42d7d04 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 10:16:27 -0400 Subject: [PATCH 163/933] [C11] Add test & update status of N1282 and DR087 Our existing diagnostics for catching unsequenced modifications handles test coverage for N1282, which is correcting the standard based on the resolution of DR087. --- clang/test/C/C11/n1282.c | 20 ++++++++++++++++++++ clang/test/C/drs/dr0xx.c | 4 ++++ clang/www/c_dr_status.html | 2 +- clang/www/c_status.html | 2 +- 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 clang/test/C/C11/n1282.c diff --git a/clang/test/C/C11/n1282.c b/clang/test/C/C11/n1282.c new file mode 100644 index 000000000000..ed952790c883 --- /dev/null +++ b/clang/test/C/C11/n1282.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -verify -Wunsequenced -Wno-unused-value %s + +/* WG14 N1282: Yes + * Clarification of Expressions + */ + +int g; + +int f(int i) { + g = i; + return 0; +} + +int main(void) { + int x; + x = (10, g = 1, 20) + (30, g = 2, 40); /* Line A */ // expected-warning {{multiple unsequenced modifications to 'g'}} + x = (10, f(1), 20) + (30, f(2), 40); /* Line B */ + x = (g = 1) + (g = 2); /* Line C */ // expected-warning {{multiple unsequenced modifications to 'g'}} + return 0; +} diff --git a/clang/test/C/drs/dr0xx.c b/clang/test/C/drs/dr0xx.c index c93cfb63d604..36de32a93da9 100644 --- a/clang/test/C/drs/dr0xx.c +++ b/clang/test/C/drs/dr0xx.c @@ -73,6 +73,10 @@ * WG14 DR085: yes * Returning from main * + * WG14 DR087: yes + * Order of evaluation + * Note: this DR is covered by C/C11/n1282.c + * * WG14 DR086: yes * Object-like macros in system headers * diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index fa2ceb1be58b..ed45123ffd0e 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -577,7 +577,7 @@ conformance.

87 NAD Order of evaluation - Unknown + Yes 88 diff --git a/clang/www/c_status.html b/clang/www/c_status.html index b1f5ab4cbc4f..0069da74cbd5 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -401,7 +401,7 @@ conformance.

Clarification of expressions N1282 - Unknown + Yes Extending the lifetime of temporary objects (factored approach) -- GitLab From 8612fa0d84c730a753d04de012a8372ba5a10677 Mon Sep 17 00:00:00 2001 From: agozillon Date: Fri, 22 Mar 2024 15:32:39 +0100 Subject: [PATCH 164/933] [MLIR][OpenMP] Refactor bounds offsetting and fix to apply to all directives (#84349) This PR refactors bounds offsetting by combining the two differing implementations (one applying to initial derived type member map implementation for descriptors and the other for regular arrays, effectively allocatable array vs regular array in fortran) now that it's a little simpler to do. The PR also moves the utilization of createAlteredByCaptureMap into genMapInfoOp, where it will be correctly applied to all MapInfoData, appropriately offsetting and altering Pointer data set in the kernel argument structure on the host. This primarily means bounds offsets will now correctly apply to enter/exit/update map clauses as opposed to just the Target directive that is currently the case. A few fortran runtime tests have been added to verify this new behavior. This PR depends on: https://github.com/llvm/llvm-project/pull/84328 and is an extraction of the larger derived type member map PR stack (so a requirement for it to land). --- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 374 +++++++++++------- ...target-fortran-allocatable-types-host.mlir | 11 +- mlir/test/Target/LLVMIR/omptarget-llvm.mlir | 25 +- .../fortran/target-map-enter-exit-array-2.f90 | 39 ++ .../target-map-enter-exit-array-bounds.f90 | 44 +++ .../fortran/target-map-enter-exit-scalar.f90 | 33 ++ 6 files changed, 360 insertions(+), 166 deletions(-) create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 7df33470ea06..646d0ed73084 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -1787,6 +1787,20 @@ getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, return suffix; } +static bool isDeclareTargetLink(mlir::Value value) { + if (auto addressOfOp = + llvm::dyn_cast_if_present(value.getDefiningOp())) { + auto modOp = addressOfOp->getParentOfType(); + Operation *gOp = modOp.lookupSymbol(addressOfOp.getGlobalName()); + if (auto declareTargetGlobal = + llvm::dyn_cast(gOp)) + if (declareTargetGlobal.getDeclareTargetCaptureClause() == + mlir::omp::DeclareTargetCaptureClause::link) + return true; + } + return false; +} + // Returns the reference pointer generated by the lowering of the declare target // operation in cases where the link clause is used or the to clause is used in // USM mode. @@ -1982,6 +1996,99 @@ void collectMapDataFromMapOperands(MapInfoData &mapData, } } +/// This function calculates the array/pointer offset for map data provided +/// with bounds operations, e.g. when provided something like the following: +/// +/// Fortran +/// map(tofrom: array(2:5, 3:2)) +/// or +/// C++ +/// map(tofrom: array[1:4][2:3]) +/// We must calculate the initial pointer offset to pass across, this function +/// performs this using bounds. +/// +/// NOTE: which while specified in row-major order it currently needs to be +/// flipped for Fortran's column order array allocation and access (as +/// opposed to C++'s row-major, hence the backwards processing where order is +/// important). This is likely important to keep in mind for the future when +/// we incorporate a C++ frontend, both frontends will need to agree on the +/// ordering of generated bounds operations (one may have to flip them) to +/// make the below lowering frontend agnostic. The offload size +/// calcualtion may also have to be adjusted for C++. +std::vector +calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder, bool isArrayTy, + mlir::OperandRange bounds) { + std::vector idx; + // There's no bounds to calculate an offset from, we can safely + // ignore and return no indices. + if (bounds.empty()) + return idx; + + // If we have an array type, then we have its type so can treat it as a + // normal GEP instruction where the bounds operations are simply indexes + // into the array. We currently do reverse order of the bounds, which + // I believe leans more towards Fortran's column-major in memory. + if (isArrayTy) { + idx.push_back(builder.getInt64(0)); + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + idx.push_back(moduleTranslation.lookupValue(boundOp.getLowerBound())); + } + } + } else { + // If we do not have an array type, but we have bounds, then we're dealing + // with a pointer that's being treated like an array and we have the + // underlying type e.g. an i32, or f64 etc, e.g. a fortran descriptor base + // address (pointer pointing to the actual data) so we must caclulate the + // offset using a single index which the following two loops attempts to + // compute. + + // Calculates the size offset we need to make per row e.g. first row or + // column only needs to be offset by one, but the next would have to be + // the previous row/column offset multiplied by the extent of current row. + // + // For example ([1][10][100]): + // + // - First row/column we move by 1 for each index increment + // - Second row/column we move by 1 (first row/column) * 10 (extent/size of + // current) for 10 for each index increment + // - Third row/column we would move by 10 (second row/column) * + // (extent/size of current) 100 for 1000 for each index increment + std::vector dimensionIndexSizeOffset{builder.getInt64(1)}; + for (size_t i = 1; i < bounds.size(); ++i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + dimensionIndexSizeOffset.push_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getExtent()), + dimensionIndexSizeOffset[i - 1])); + } + } + + // Now that we have calculated how much we move by per index, we must + // multiply each lower bound offset in indexes by the size offset we + // have calculated in the previous and accumulate the results to get + // our final resulting offset. + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + if (idx.empty()) + idx.emplace_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + else + idx.back() = builder.CreateAdd( + idx.back(), builder.CreateMul(moduleTranslation.lookupValue( + boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + } + } + } + + return idx; +} + // This creates two insertions into the MapInfosTy data structure for the // "parent" of a set of members, (usually a container e.g. // class/structure/derived type) when subsequent members have also been @@ -2057,6 +2164,27 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers( return memberOfFlag; } +// The intent is to verify if the mapped data being passed is a +// pointer -> pointee that requires special handling in certain cases, +// e.g. applying the OMP_MAP_PTR_AND_OBJ map type. +// +// There may be a better way to verify this, but unfortunately with +// opaque pointers we lose the ability to easily check if something is +// a pointer whilst maintaining access to the underlying type. +static bool checkIfPointerMap(mlir::omp::MapInfoOp mapOp) { + // If we have a varPtrPtr field assigned then the underlying type is a pointer + if (mapOp.getVarPtrPtr()) + return true; + + // If the map data is declare target with a link clause, then it's represented + // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has + // no relation to pointers. + if (isDeclareTargetLink(mapOp.getVarPtr())) + return true; + + return false; +} + // This function is intended to add explicit mappings of members static void processMapMembersWithParent( LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, @@ -2083,8 +2211,11 @@ static void processMapMembersWithParent( auto mapFlag = llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value()); mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF; ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag); - mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + if (checkIfPointerMap(memberClause)) + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + combinedInfo.Types.emplace_back(mapFlag); combinedInfo.DevicePointers.emplace_back( llvm::OpenMPIRBuilder::DeviceInfoTy::None); @@ -2092,55 +2223,7 @@ static void processMapMembersWithParent( LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder)); combinedInfo.BasePointers.emplace_back(mapData.BasePointers[memberDataIdx]); - - std::vector idx{builder.getInt64(0)}; - llvm::Value *offsetAddress = nullptr; - if (!memberClause.getBounds().empty()) { - if (mapData.BaseType[memberDataIdx]->isArrayTy()) { - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - } else { - std::vector dimensionIndexSizeOffset{ - builder.getInt64(1)}; - for (size_t i = 1; i < memberClause.getBounds().size(); ++i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - dimensionIndexSizeOffset.push_back(builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getExtent()), - dimensionIndexSizeOffset[i - 1])); - } - } - - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - if (!offsetAddress) - offsetAddress = builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i]); - else - offsetAddress = builder.CreateAdd( - offsetAddress, - builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i])); - } - } - } - } - - llvm::Value *memberIdx = - builder.CreateLoad(builder.getPtrTy(), mapData.Pointers[memberDataIdx]); - memberIdx = builder.CreateInBoundsGEP( - mapData.BaseType[memberDataIdx], memberIdx, - offsetAddress ? std::vector{offsetAddress} : idx, - "member_idx"); - combinedInfo.Pointers.emplace_back(memberIdx); + combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]); combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]); } } @@ -2158,6 +2241,76 @@ static void processMapWithMembersOf( memberOfParentFlag); } +// This is a variation on Clang's GenerateOpenMPCapturedVars, which +// generates different operation (e.g. load/store) combinations for +// arguments to the kernel, based on map capture kinds which are then +// utilised in the combinedInfo in place of the original Map value. +static void +createAlteredByCaptureMap(MapInfoData &mapData, + LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder) { + for (size_t i = 0; i < mapData.MapClause.size(); ++i) { + // if it's declare target, skip it, it's handled seperately. + if (!mapData.IsDeclareTarget[i]) { + auto mapOp = + mlir::dyn_cast_if_present(mapData.MapClause[i]); + mlir::omp::VariableCaptureKind captureKind = + mapOp.getMapCaptureType().value_or( + mlir::omp::VariableCaptureKind::ByRef); + bool isPtrTy = checkIfPointerMap(mapOp); + + // Currently handles array sectioning lowerbound case, but more + // logic may be required in the future. Clang invokes EmitLValue, + // which has specialised logic for special Clang types such as user + // defines, so it is possible we will have to extend this for + // structures or other complex types. As the general idea is that this + // function mimics some of the logic from Clang that we require for + // kernel argument passing from host -> device. + switch (captureKind) { + case mlir::omp::VariableCaptureKind::ByRef: { + llvm::Value *newV = mapData.Pointers[i]; + std::vector offsetIdx = calculateBoundsOffset( + moduleTranslation, builder, mapData.BaseType[i]->isArrayTy(), + mapOp.getBounds()); + if (isPtrTy) + newV = builder.CreateLoad(builder.getPtrTy(), newV); + + if (!offsetIdx.empty()) + newV = builder.CreateInBoundsGEP(mapData.BaseType[i], newV, offsetIdx, + "array_offset"); + mapData.Pointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::ByCopy: { + llvm::Type *type = mapData.BaseType[i]; + llvm::Value *newV; + if (mapData.Pointers[i]->getType()->isPointerTy()) + newV = builder.CreateLoad(type, mapData.Pointers[i]); + else + newV = mapData.Pointers[i]; + + if (!isPtrTy) { + auto curInsert = builder.saveIP(); + builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); + auto *memTempAlloc = + builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); + builder.restoreIP(curInsert); + + builder.CreateStore(newV, memTempAlloc); + newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); + } + + mapData.Pointers[i] = newV; + mapData.BasePointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::This: + case mlir::omp::VariableCaptureKind::VLAType: + mapData.MapClause[i]->emitOpError("Unhandled capture kind"); + break; + } + } + } +} + // Generate all map related information and fill the combinedInfo. static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, @@ -2167,6 +2320,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, const SmallVector &devPtrOperands = {}, const SmallVector &devAddrOperands = {}, bool isTargetParams = false) { + // We wish to modify some of the methods in which arguments are + // passed based on their capture type by the target region, this can + // involve generating new loads and stores, which changes the + // MLIR value to LLVM value mapping, however, we only wish to do this + // locally for the current function/target and also avoid altering + // ModuleTranslation, so we remap the base pointer or pointer stored + // in the map infos corresponding MapInfoData, which is later accessed + // by genMapInfos and createTarget to help generate the kernel and + // kernel arg structure. It primarily becomes relevant in cases like + // bycopy, or byref range'd arrays. In the default case, we simply + // pass thee pointer byref as both basePointer and pointer. + if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) + createAlteredByCaptureMap(mapData, moduleTranslation, builder); + llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); auto fail = [&combinedInfo]() -> void { @@ -2200,19 +2367,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, continue; } - // Declare Target Mappings are excluded from being marked as - // OMP_MAP_TARGET_PARAM as they are not passed as parameters, they're - // marked with OMP_MAP_PTR_AND_OBJ instead. auto mapFlag = mapData.Types[i]; - if (mapData.IsDeclareTarget[i]) + bool isPtrTy = checkIfPointerMap(mapInfoOp); + if (isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; - else if (isTargetParams) + + // Declare Target Mappings are excluded from being marked as + // OMP_MAP_TARGET_PARAM as they are not passed as parameters. + if (isTargetParams && !mapData.IsDeclareTarget[i]) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; if (auto mapInfoOp = dyn_cast(mapData.MapClause[i])) if (mapInfoOp.getMapCaptureType().value() == mlir::omp::VariableCaptureKind::ByCopy && - !mapInfoOp.getVarType().isa()) + !isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]); @@ -2662,86 +2830,6 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, return builder.saveIP(); } -// This is a variation on Clang's GenerateOpenMPCapturedVars, which -// generates different operation (e.g. load/store) combinations for -// arguments to the kernel, based on map capture kinds which are then -// utilised in the combinedInfo in place of the original Map value. -static void -createAlteredByCaptureMap(MapInfoData &mapData, - LLVM::ModuleTranslation &moduleTranslation, - llvm::IRBuilderBase &builder) { - for (size_t i = 0; i < mapData.MapClause.size(); ++i) { - // if it's declare target, skip it, it's handled seperately. - if (!mapData.IsDeclareTarget[i]) { - mlir::omp::VariableCaptureKind captureKind = - mlir::omp::VariableCaptureKind::ByRef; - - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - captureKind = mapOp.getMapCaptureType().value_or( - mlir::omp::VariableCaptureKind::ByRef); - } - - switch (captureKind) { - case mlir::omp::VariableCaptureKind::ByRef: { - // Currently handles array sectioning lowerbound case, but more - // logic may be required in the future. Clang invokes EmitLValue, - // which has specialised logic for special Clang types such as user - // defines, so it is possible we will have to extend this for - // structures or other complex types. As the general idea is that this - // function mimics some of the logic from Clang that we require for - // kernel argument passing from host -> device. - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - if (!mapOp.getBounds().empty() && mapData.BaseType[i]->isArrayTy()) { - - std::vector idx = - std::vector{builder.getInt64(0)}; - for (int i = mapOp.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = - mlir::dyn_cast_if_present( - mapOp.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - - mapData.Pointers[i] = builder.CreateInBoundsGEP( - mapData.BaseType[i], mapData.Pointers[i], idx); - } - } - } break; - case mlir::omp::VariableCaptureKind::ByCopy: { - llvm::Type *type = mapData.BaseType[i]; - llvm::Value *newV; - if (mapData.Pointers[i]->getType()->isPointerTy()) - newV = builder.CreateLoad(type, mapData.Pointers[i]); - else - newV = mapData.Pointers[i]; - - if (!type->isPointerTy()) { - auto curInsert = builder.saveIP(); - builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); - auto *memTempAlloc = - builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); - builder.restoreIP(curInsert); - - builder.CreateStore(newV, memTempAlloc); - newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); - } - - mapData.Pointers[i] = newV; - mapData.BasePointers[i] = newV; - } break; - case mlir::omp::VariableCaptureKind::This: - case mlir::omp::VariableCaptureKind::VLAType: - mapData.MapClause[i]->emitOpError("Unhandled capture kind"); - break; - } - } - } -} - static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { @@ -2810,20 +2898,6 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, collectMapDataFromMapOperands(mapData, mapOperands, moduleTranslation, dl, builder); - // We wish to modify some of the methods in which kernel arguments are - // passed based on their capture type by the target region, this can - // involve generating new loads and stores, which changes the - // MLIR value to LLVM value mapping, however, we only wish to do this - // locally for the current function/target and also avoid altering - // ModuleTranslation, so we remap the base pointer or pointer stored - // in the map infos corresponding MapInfoData, which is later accessed - // by genMapInfos and createTarget to help generate the kernel and - // kernel arg structure. It primarily becomes relevant in cases like - // bycopy, or byref range'd arrays. In the default case, we simply - // pass thee pointer byref as both basePointer and pointer. - if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) - createAlteredByCaptureMap(mapData, moduleTranslation, builder); - llvm::OpenMPIRBuilder::MapInfosTy combinedInfos; auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & { diff --git a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir index e8c388627a0a..7cb22dbb10b1 100644 --- a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir @@ -26,7 +26,7 @@ module attributes {omp.is_target_device = false} { %14 = llvm.sub %11, %2 : i64 %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%14 : i64) extent(%11 : i64) stride(%13 : i64) start_idx(%9 : i64) {stride_in_bytes = true} %16 = llvm.getelementptr %3[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> - %17 = omp.map.info var_ptr(%16 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} + %17 = omp.map.info var_ptr(%3 : !llvm.ptr, f32) var_ptr_ptr(%16 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} %18 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%17 : !llvm.ptr) -> !llvm.ptr {name = "full_arr"} %19 = llvm.getelementptr %6[0, 7, %7, 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> %20 = llvm.load %19 : !llvm.ptr -> i64 @@ -81,20 +81,19 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[ARR_SECT_SIZE2:.*]] = add i64 %[[ARR_SECT_SIZE3]], 1 // CHECK: %[[ARR_SECT_SIZE1:.*]] = mul i64 1, %[[ARR_SECT_SIZE2]] // CHECK: %[[ARR_SECT_SIZE:.*]] = mul i64 %[[ARR_SECT_SIZE1]], 4 -// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[LFULL_ARR:.*]] = load ptr, ptr @_QFEfull_arr, align 8 // CHECK: %[[FULL_ARR_PTR:.*]] = getelementptr inbounds float, ptr %[[LFULL_ARR]], i64 0 -// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[ARR_SECT_OFFSET1:.*]] = mul i64 %[[ARR_SECT_OFFSET2]], 1 // CHECK: %[[LARR_SECT:.*]] = load ptr, ptr @_QFEsect_arr, align 8 // CHECK: %[[ARR_SECT_PTR:.*]] = getelementptr inbounds i32, ptr %[[LARR_SECT]], i64 %[[ARR_SECT_OFFSET1]] +// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 +// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) +// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[SCALAR_DESC_SZ4:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8 }, ptr %[[SCALAR_ALLOCA]], i32 1 // CHECK: %[[SCALAR_DESC_SZ3:.*]] = ptrtoint ptr %[[SCALAR_DESC_SZ4]] to i64 // CHECK: %[[SCALAR_DESC_SZ2:.*]] = ptrtoint ptr %[[SCALAR_ALLOCA]] to i64 // CHECK: %[[SCALAR_DESC_SZ1:.*]] = sub i64 %[[SCALAR_DESC_SZ3]], %[[SCALAR_DESC_SZ2]] // CHECK: %[[SCALAR_DESC_SZ:.*]] = sdiv exact i64 %[[SCALAR_DESC_SZ1]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) -// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 -// CHECK: %[[SCALAR_PTR:.*]] = getelementptr inbounds float, ptr %[[SCALAR_PTR_LOAD]], i64 0 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 0 // CHECK: store ptr @_QFEfull_arr, ptr %[[OFFLOADBASEPTRS]], align 8 @@ -145,4 +144,4 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 8 // CHECK: store ptr %[[SCALAR_BASE]], ptr %[[OFFLOADBASEPTRS]], align 8 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_ptrs, i32 0, i32 8 -// CHECK: store ptr %[[SCALAR_PTR]], ptr %[[OFFLOADPTRS]], align 8 +// CHECK: store ptr %[[SCALAR_PTR_LOAD]], ptr %[[OFFLOADPTRS]], align 8 diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir index 4b1d5d58f14e..2f629675442d 100644 --- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir @@ -66,16 +66,17 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) { // CHECK: %[[VAL_2:.*]] = alloca [1 x ptr], align 8 // CHECK: br label %[[VAL_3:.*]] // CHECK: entry: ; preds = %[[VAL_4:.*]] +// CHECK: %[[ARR_OFFSET:.*]] = getelementptr inbounds [1024 x i32], ptr %[[ARR_DATA:.*]], i64 0, i64 0 // CHECK: %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6:.*]], ptr %[[VAL_5]], align 8 +// CHECK: store ptr %[[ARR_DATA]], ptr %[[VAL_5]], align 8 // CHECK: %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6]], ptr %[[VAL_7]], align 8 +// CHECK: store ptr %[[ARR_OFFSET]], ptr %[[VAL_7]], align 8 // CHECK: %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_8]], align 8 // CHECK: %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null) -// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_6]], i32 0, i64 0 +// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[ARR_DATA]], i32 0, i64 0 // CHECK: store i32 99, ptr %[[VAL_11]], align 4 // CHECK: %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 @@ -153,16 +154,18 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: entry: ; preds = %[[VAL_12:.*]] // CHECK: br i1 %[[VAL_9]], label %[[VAL_13:.*]], label %[[VAL_14:.*]] // CHECK: omp_if.then: ; preds = %[[VAL_11]] +// CHECK: %[[ARR_OFFSET1:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16:.*]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET2:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20:.*]], i64 0, i64 0 // CHECK: %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16:.*]], ptr %[[VAL_15]], align 8 // CHECK: %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_17]], align 8 +// CHECK: store ptr %[[ARR_OFFSET1]], ptr %[[VAL_17]], align 8 // CHECK: %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_18]], align 8 // CHECK: %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20:.*]], ptr %[[VAL_19]], align 8 // CHECK: %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8 +// CHECK: store ptr %[[ARR_OFFSET2]], ptr %[[VAL_21]], align 8 // CHECK: %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_22]], align 8 // CHECK: %[[VAL_23:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 @@ -176,26 +179,28 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: %[[VAL_27:.*]] = icmp sgt i32 %[[VAL_26]], 10 // CHECK: %[[VAL_28:.*]] = load i32, ptr %[[VAL_6]], align 4 // CHECK: br i1 %[[VAL_27]], label %[[VAL_29:.*]], label %[[VAL_30:.*]] -// CHECK: omp_if.then1: ; preds = %[[VAL_25]] +// CHECK: omp_if.then2: ; preds = %[[VAL_25]] +// CHECK: %[[ARR_OFFSET3:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET4:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20]], i64 0, i64 0 // CHECK: %[[VAL_31:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16]], ptr %[[VAL_31]], align 8 // CHECK: %[[VAL_32:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_32]], align 8 +// CHECK: store ptr %[[ARR_OFFSET3]], ptr %[[VAL_32]], align 8 // CHECK: %[[VAL_33:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_33]], align 8 // CHECK: %[[VAL_34:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20]], ptr %[[VAL_34]], align 8 // CHECK: %[[VAL_35:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_35]], align 8 +// CHECK: store ptr %[[ARR_OFFSET4]], ptr %[[VAL_35]], align 8 // CHECK: %[[VAL_36:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_36]], align 8 // CHECK: %[[VAL_37:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_38:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_end_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_37]], ptr %[[VAL_38]], ptr @.offload_sizes.1, ptr @.offload_maptypes.2, ptr @.offload_mapnames.3, ptr null) // CHECK: br label %[[VAL_39:.*]] -// CHECK: omp_if.else5: ; preds = %[[VAL_25]] +// CHECK: omp_if.else8: ; preds = %[[VAL_25]] // CHECK: br label %[[VAL_39]] -// CHECK: omp_if.end6: ; preds = %[[VAL_30]], %[[VAL_29]] +// CHECK: omp_if.end9: ; preds = %[[VAL_30]], %[[VAL_29]] // CHECK: ret void // ----- diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 new file mode 100644 index 000000000000..489c2532a762 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 @@ -0,0 +1,39 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array) + + ! Shouldn't overwrite data already locked in + ! on target via enter, this will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + !$omp target + do i=1,10 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array) + + print*, array +end program + +!CHECK: 3 6 9 12 15 18 21 24 27 30 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 new file mode 100644 index 000000000000..3c8c3507ed72 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 @@ -0,0 +1,44 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! with specified bounds +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic + +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array(3:6)) + + ! Shouldn't overwrite data already locked in + ! on target via enter, which will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + ! The compiler/runtime is less lenient about read/write out of + ! bounds when using enter and exit, we have to specifically loop + ! over the correctly mapped range + !$omp target + do i=3,6 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array(3:6)) + + print *, array +end program + +!CHECK: 10 10 9 12 15 18 10 10 10 10 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 new file mode 100644 index 000000000000..29a0b5ee3e62 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 @@ -0,0 +1,33 @@ +! Offloading test checking interaction of an +! enter and exit map of an scalar +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: scalar + scalar = 10 + + !$omp target enter data map(to: scalar) + + !ignored, as we've already attached + scalar = 20 + + !$omp target + scalar = scalar + 50 + !$omp end target + + !$omp target exit data map(from: scalar) + + ! not the answer one may expect, but it is the same + ! answer Clang gives so we are correctly on par with + ! Clang for the moment. + print *, scalar +end program + +!CHECK: 10 -- GitLab From 71db97152173a524a3e16e02b7fdc50f405c8695 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 22 Mar 2024 15:39:52 +0100 Subject: [PATCH 165/933] [mlir][emitc] Arith to EmitC: Handle addi, subi and muli (#86120) Important to consider that `arith` has wrap around semantics, and in C++ signed overflow is UB. Unless the operation guarantees that no signed overflow happens, we will perform the arithmetic in an equivalent unsigned type. `bool` also doesn't wrap around in C++, and is not addressed here. --- .../Conversion/ArithToEmitC/ArithToEmitC.cpp | 52 +++++++++++++++++++ .../ArithToEmitC/arith-to-emitc-failed.mlir | 15 ++++++ .../ArithToEmitC/arith-to-emitc.mlir | 51 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp index 3532785c31b9..db493c1294ba 100644 --- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp @@ -55,6 +55,55 @@ public: } }; +template +class IntegerOpConversion final : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Type type = this->getTypeConverter()->convertType(op.getType()); + if (!isa_and_nonnull(type)) { + return rewriter.notifyMatchFailure(op, "expected integer type"); + } + + if (type.isInteger(1)) { + // arith expects wrap-around arithmethic, which doesn't happen on `bool`. + return rewriter.notifyMatchFailure(op, "i1 type is not implemented"); + } + + Value lhs = adaptor.getLhs(); + Value rhs = adaptor.getRhs(); + Type arithmeticType = type; + if ((type.isSignlessInteger() || type.isSignedInteger()) && + !bitEnumContainsAll(op.getOverflowFlags(), + arith::IntegerOverflowFlags::nsw)) { + // If the C type is signed and the op doesn't guarantee "No Signed Wrap", + // we compute in unsigned integers to avoid UB. + arithmeticType = rewriter.getIntegerType(type.getIntOrFloatBitWidth(), + /*isSigned=*/false); + } + if (arithmeticType != type) { + lhs = rewriter.template create(op.getLoc(), arithmeticType, + lhs); + rhs = rewriter.template create(op.getLoc(), arithmeticType, + rhs); + } + + Value result = rewriter.template create(op.getLoc(), + arithmeticType, lhs, rhs); + + if (arithmeticType != type) { + result = + rewriter.template create(op.getLoc(), type, result); + } + rewriter.replaceOp(op, result); + return success(); + } +}; + class SelectOpConversion : public OpConversionPattern { public: using OpConversionPattern::OpConversionPattern; @@ -96,6 +145,9 @@ void mlir::populateArithToEmitCPatterns(TypeConverter &typeConverter, ArithOpConversion, ArithOpConversion, ArithOpConversion, + IntegerOpConversion, + IntegerOpConversion, + IntegerOpConversion, SelectOpConversion >(typeConverter, ctx); // clang-format on diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir new file mode 100644 index 000000000000..30abd81f3d44 --- /dev/null +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir @@ -0,0 +1,15 @@ +// RUN: mlir-opt -convert-arith-to-emitc %s -split-input-file -verify-diagnostics + +func.func @bool(%arg0: i1, %arg1: i1) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : i1 + return +} + +// ----- + +func.func @vector(%arg0: vector<4xi32>, %arg1: vector<4xi32>) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : vector<4xi32> + return +} diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir index 022530ef4db8..76ba518577ab 100644 --- a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir @@ -37,6 +37,57 @@ func.func @arith_ops(%arg0: f32, %arg1: f32) { // ----- +// CHECK-LABEL: arith_integer_ops +func.func @arith_integer_ops(%arg0: i32, %arg1: i32) { + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[ADD:[^ ]*]] = emitc.add %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[ADD]] : ui32 to i32 + %0 = arith.addi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[SUB:[^ ]*]] = emitc.sub %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[SUB]] : ui32 to i32 + %1 = arith.subi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[MUL:[^ ]*]] = emitc.mul %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[MUL]] : ui32 to i32 + %2 = arith.muli %arg0, %arg1 : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_integer_ops_signed_nsw +func.func @arith_integer_ops_signed_nsw(%arg0: i32, %arg1: i32) { + // CHECK: emitc.add %arg0, %arg1 : (i32, i32) -> i32 + %0 = arith.addi %arg0, %arg1 overflow : i32 + // CHECK: emitc.sub %arg0, %arg1 : (i32, i32) -> i32 + %1 = arith.subi %arg0, %arg1 overflow : i32 + // CHECK: emitc.mul %arg0, %arg1 : (i32, i32) -> i32 + %2 = arith.muli %arg0, %arg1 overflow : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_index +func.func @arith_index(%arg0: index, %arg1: index) { + // CHECK: emitc.add %arg0, %arg1 : (index, index) -> index + %0 = arith.addi %arg0, %arg1 : index + // CHECK: emitc.sub %arg0, %arg1 : (index, index) -> index + %1 = arith.subi %arg0, %arg1 : index + // CHECK: emitc.mul %arg0, %arg1 : (index, index) -> index + %2 = arith.muli %arg0, %arg1 : index + + return +} + +// ----- + func.func @arith_select(%arg0: i1, %arg1: tensor<8xi32>, %arg2: tensor<8xi32>) -> () { // CHECK: [[V0:[^ ]*]] = emitc.conditional %arg0, %arg1, %arg2 : tensor<8xi32> %0 = arith.select %arg0, %arg1, %arg2 : i1, tensor<8xi32> -- GitLab From 9c0a0659d40f613e873e416833d2293365b48e06 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Fri, 22 Mar 2024 07:31:35 -0700 Subject: [PATCH 166/933] [SLP]Fix a crash for non-profitable non-schedulable single buildvector node tree, if the threshold allows its vectorization. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 + .../small-tree-not-schedulable-bv-node.ll | 263 ++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7295ae0ba90b..0f7afa2fc25c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9278,6 +9278,8 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { bool IsAllowedSingleBVNode = VectorizableTree.size() > 1 || (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() && + VectorizableTree.front()->getOpcode() != Instruction::PHI && + VectorizableTree.front()->getOpcode() != Instruction::GetElementPtr && allSameBlock(VectorizableTree.front()->Scalars)); if (any_of(VectorizableTree, [&](const std::unique_ptr &TE) { return TE->State == TreeEntry::NeedToGather && diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll new file mode 100644 index 000000000000..26f3fcae3a33 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll @@ -0,0 +1,263 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=riscv64-unknown-linux-gnu -slp-threshold=-100 -mattr=+v < %s | FileCheck %s + +define void @test1() personality ptr null { +; CHECK-LABEL: define void @test1( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] personality ptr null { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL33:%.*]] = invoke ptr null(i64 0, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT32:%.*]] unwind label [[LPAD31_LOOPEXIT:%.*]] +; CHECK: invoke.cont32: +; CHECK-NEXT: invoke void null(ptr null, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT37:%.*]] unwind label [[LPAD34_LOOPEXIT:%.*]] +; CHECK: invoke.cont37: +; CHECK-NEXT: unreachable +; CHECK: lpad31.loopexit: +; CHECK-NEXT: [[LPAD_LOOPEXIT:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[EHCLEANUP47:%.*]] +; CHECK: lpad34.loopexit: +; CHECK-NEXT: [[DOTLCSSA101:%.*]] = phi ptr [ null, [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[CALL33_LCSSA96:%.*]] = phi ptr [ [[CALL33]], [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[LPAD_LOOPEXIT56:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY:%.*]] +; CHECK: lpad34.loopexit.split-lp: +; CHECK-NEXT: [[LPAD_LOOPEXIT_SPLIT_LP57:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY]] +; CHECK: lpad34.body: +; CHECK-NEXT: [[TMP0:%.*]] = phi ptr [ [[DOTLCSSA101]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP:%.*]] ] +; CHECK-NEXT: [[CALL3399:%.*]] = phi ptr [ [[CALL33_LCSSA96]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP]] ] +; CHECK-NEXT: br label [[EHCLEANUP47]] +; CHECK: ehcleanup47: +; CHECK-NEXT: resume { ptr, i32 } zeroinitializer +; +entry: + %call33 = invoke ptr null(i64 0, ptr null) + to label %invoke.cont32 unwind label %lpad31.loopexit + +invoke.cont32: + invoke void null(ptr null, ptr null) + to label %invoke.cont37 unwind label %lpad34.loopexit + +invoke.cont37: + unreachable + +lpad31.loopexit: + %lpad.loopexit = landingpad { ptr, i32 } + cleanup + br label %ehcleanup47 + +lpad34.loopexit: + %.lcssa101 = phi ptr [ null, %invoke.cont32 ] + %call33.lcssa96 = phi ptr [ %call33, %invoke.cont32 ] + %lpad.loopexit56 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.loopexit.split-lp: + %lpad.loopexit.split-lp57 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.body: + %0 = phi ptr [ %.lcssa101, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + %call3399 = phi ptr [ %call33.lcssa96, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + br label %ehcleanup47 + +ehcleanup47: + resume { ptr, i32 } zeroinitializer +} + +define i32 @test2(i64 %idx.ext.i48.pre-phi) { +; CHECK-LABEL: define i32 @test2( +; CHECK-SAME: i64 [[IDX_EXT_I48_PRE_PHI:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[DO_ACTION:%.*]] +; CHECK: do_action: +; CHECK-NEXT: switch i32 0, label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 1, label [[CLEANUP185:%.*]] +; CHECK-NEXT: i32 2, label [[CLEANUP185]] +; CHECK-NEXT: i32 0, label [[CLEANUP185]] +; CHECK-NEXT: i32 4, label [[CLEANUP185]] +; CHECK-NEXT: i32 5, label [[CLEANUP185]] +; CHECK-NEXT: i32 6, label [[CLEANUP185]] +; CHECK-NEXT: i32 7, label [[CLEANUP185]] +; CHECK-NEXT: i32 8, label [[CLEANUP185]] +; CHECK-NEXT: i32 9, label [[CLEANUP185]] +; CHECK-NEXT: i32 10, label [[CLEANUP185]] +; CHECK-NEXT: i32 11, label [[CLEANUP185]] +; CHECK-NEXT: i32 12, label [[CLEANUP185]] +; CHECK-NEXT: i32 13, label [[CLEANUP185]] +; CHECK-NEXT: i32 14, label [[CLEANUP185]] +; CHECK-NEXT: i32 15, label [[CLEANUP185]] +; CHECK-NEXT: i32 16, label [[CLEANUP185]] +; CHECK-NEXT: i32 17, label [[CLEANUP185]] +; CHECK-NEXT: i32 18, label [[CLEANUP185]] +; CHECK-NEXT: i32 19, label [[CLEANUP185]] +; CHECK-NEXT: i32 20, label [[CLEANUP185]] +; CHECK-NEXT: i32 21, label [[CLEANUP185]] +; CHECK-NEXT: i32 22, label [[CLEANUP185]] +; CHECK-NEXT: i32 23, label [[CLEANUP185]] +; CHECK-NEXT: i32 24, label [[CLEANUP185]] +; CHECK-NEXT: i32 25, label [[CLEANUP185]] +; CHECK-NEXT: i32 26, label [[CLEANUP185]] +; CHECK-NEXT: i32 27, label [[CLEANUP185]] +; CHECK-NEXT: i32 28, label [[CLEANUP185]] +; CHECK-NEXT: i32 29, label [[CLEANUP185]] +; CHECK-NEXT: i32 30, label [[CLEANUP185]] +; CHECK-NEXT: i32 31, label [[CLEANUP185]] +; CHECK-NEXT: i32 32, label [[CLEANUP185]] +; CHECK-NEXT: i32 33, label [[CLEANUP185]] +; CHECK-NEXT: i32 34, label [[CLEANUP185]] +; CHECK-NEXT: i32 35, label [[CLEANUP185]] +; CHECK-NEXT: i32 36, label [[CLEANUP185]] +; CHECK-NEXT: i32 37, label [[CLEANUP185]] +; CHECK-NEXT: i32 38, label [[CLEANUP185]] +; CHECK-NEXT: i32 39, label [[CLEANUP185]] +; CHECK-NEXT: i32 40, label [[CLEANUP185]] +; CHECK-NEXT: i32 41, label [[CLEANUP185]] +; CHECK-NEXT: i32 42, label [[CLEANUP185]] +; CHECK-NEXT: i32 43, label [[CLEANUP185]] +; CHECK-NEXT: i32 44, label [[CLEANUP185]] +; CHECK-NEXT: i32 45, label [[CLEANUP185]] +; CHECK-NEXT: i32 46, label [[CLEANUP185]] +; CHECK-NEXT: i32 47, label [[CLEANUP185]] +; CHECK-NEXT: i32 48, label [[CLEANUP185]] +; CHECK-NEXT: i32 49, label [[CLEANUP185]] +; CHECK-NEXT: i32 50, label [[CLEANUP185]] +; CHECK-NEXT: i32 51, label [[CLEANUP185]] +; CHECK-NEXT: i32 52, label [[CLEANUP185]] +; CHECK-NEXT: i32 53, label [[CLEANUP185]] +; CHECK-NEXT: i32 54, label [[CLEANUP185]] +; CHECK-NEXT: i32 55, label [[CLEANUP185]] +; CHECK-NEXT: i32 56, label [[CLEANUP185]] +; CHECK-NEXT: i32 57, label [[DO_ACTION]] +; CHECK-NEXT: i32 58, label [[CLEANUP185]] +; CHECK-NEXT: i32 59, label [[CLEANUP185]] +; CHECK-NEXT: i32 60, label [[DO_ACTION]] +; CHECK-NEXT: i32 61, label [[DO_ACTION]] +; CHECK-NEXT: i32 62, label [[CLEANUP185]] +; CHECK-NEXT: i32 70, label [[SW_BB175:%.*]] +; CHECK-NEXT: i32 64, label [[CLEANUP185]] +; CHECK-NEXT: i32 65, label [[DO_ACTION]] +; CHECK-NEXT: i32 66, label [[DO_ACTION]] +; CHECK-NEXT: i32 67, label [[CLEANUP185]] +; CHECK-NEXT: i32 72, label [[CLEANUP185]] +; CHECK-NEXT: i32 69, label [[DO_ACTION]] +; CHECK-NEXT: i32 71, label [[CLEANUP185]] +; CHECK-NEXT: ] +; CHECK: yy_get_previous_state.exit.loopexit: +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE:%.*]] +; CHECK: yy_find_action.backedge: +; CHECK-NEXT: [[YY_BP_1_BE:%.*]] = phi ptr [ [[ADD_PTR_I49:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT:%.*]] ] +; CHECK-NEXT: [[YY_CP_2_BE:%.*]] = phi ptr [ [[ARRAYIDX178:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT]] ] +; CHECK-NEXT: br label [[DO_ACTION]] +; CHECK: sw.bb175: +; CHECK-NEXT: [[ARRAYIDX178]] = getelementptr i8, ptr null, i64 0 +; CHECK-NEXT: [[ADD_PTR_I49]] = getelementptr i8, ptr null, i64 [[IDX_EXT_I48_PRE_PHI]] +; CHECK-NEXT: [[CMP5_I50:%.*]] = icmp ult ptr [[ADD_PTR_I49]], [[ARRAYIDX178]] +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE]] +; CHECK: sw.default: +; CHECK-NEXT: unreachable +; CHECK: cleanup185: +; CHECK-NEXT: ret i32 0 +; +entry: + br label %do_action + +do_action: + switch i32 0, label %sw.default [ + i32 1, label %cleanup185 + i32 2, label %cleanup185 + i32 0, label %cleanup185 + i32 4, label %cleanup185 + i32 5, label %cleanup185 + i32 6, label %cleanup185 + i32 7, label %cleanup185 + i32 8, label %cleanup185 + i32 9, label %cleanup185 + i32 10, label %cleanup185 + i32 11, label %cleanup185 + i32 12, label %cleanup185 + i32 13, label %cleanup185 + i32 14, label %cleanup185 + i32 15, label %cleanup185 + i32 16, label %cleanup185 + i32 17, label %cleanup185 + i32 18, label %cleanup185 + i32 19, label %cleanup185 + i32 20, label %cleanup185 + i32 21, label %cleanup185 + i32 22, label %cleanup185 + i32 23, label %cleanup185 + i32 24, label %cleanup185 + i32 25, label %cleanup185 + i32 26, label %cleanup185 + i32 27, label %cleanup185 + i32 28, label %cleanup185 + i32 29, label %cleanup185 + i32 30, label %cleanup185 + i32 31, label %cleanup185 + i32 32, label %cleanup185 + i32 33, label %cleanup185 + i32 34, label %cleanup185 + i32 35, label %cleanup185 + i32 36, label %cleanup185 + i32 37, label %cleanup185 + i32 38, label %cleanup185 + i32 39, label %cleanup185 + i32 40, label %cleanup185 + i32 41, label %cleanup185 + i32 42, label %cleanup185 + i32 43, label %cleanup185 + i32 44, label %cleanup185 + i32 45, label %cleanup185 + i32 46, label %cleanup185 + i32 47, label %cleanup185 + i32 48, label %cleanup185 + i32 49, label %cleanup185 + i32 50, label %cleanup185 + i32 51, label %cleanup185 + i32 52, label %cleanup185 + i32 53, label %cleanup185 + i32 54, label %cleanup185 + i32 55, label %cleanup185 + i32 56, label %cleanup185 + i32 57, label %do_action + i32 58, label %cleanup185 + i32 59, label %cleanup185 + i32 60, label %do_action + i32 61, label %do_action + i32 62, label %cleanup185 + i32 70, label %sw.bb175 + i32 64, label %cleanup185 + i32 65, label %do_action + i32 66, label %do_action + i32 67, label %cleanup185 + i32 72, label %cleanup185 + i32 69, label %do_action + i32 71, label %cleanup185 + ] + +yy_get_previous_state.exit.loopexit: + br label %yy_find_action.backedge + +yy_find_action.backedge: + %yy_bp.1.be = phi ptr [ %add.ptr.i49, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + %yy_cp.2.be = phi ptr [ %arrayidx178, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + br label %do_action + +sw.bb175: + %arrayidx178 = getelementptr i8, ptr null, i64 0 + %add.ptr.i49 = getelementptr i8, ptr null, i64 %idx.ext.i48.pre-phi + %cmp5.i50 = icmp ult ptr %add.ptr.i49, %arrayidx178 + br label %yy_find_action.backedge + +sw.default: + unreachable + +cleanup185: + ret i32 0 +} -- GitLab From 6f44bb7717897191be25aa01161831c67cdf5b84 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 15:40:15 +0100 Subject: [PATCH 167/933] [Object] Ensure header size not to underflow in `OffloadBinary::create` Prevent potential integer underflows when header size is not valid. Fixes: https://github.com/llvm/llvm-project/issues/86280. --- llvm/lib/Object/OffloadBinary.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 4ab6536dc90b..6e9f8bed513c 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -189,7 +189,10 @@ OffloadBinary::create(MemoryBufferRef Buf) { return errorCodeToError(object_error::parse_failed); if (TheHeader->Size > Buf.getBufferSize() || - TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || + TheHeader->Size < sizeof(Entry) || TheHeader->Size < sizeof(Header)) + return errorCodeToError(object_error::unexpected_eof); + + if (TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || TheHeader->EntrySize > TheHeader->Size - sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); -- GitLab From b44771f480385fa93ba7719a57e759e19747e709 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 168/933] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. This is recommitted as 66f88de was reverted because of failures caused by lacking `--target` option. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 324 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 389 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 45b2e01af997..d6e179ca9d69 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..0227487015ba --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,324 @@ +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvi20u32 \ +// RUN: | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvi20u64 \ +// RUN: | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20u64 \ +// RUN: | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20s64 \ +// RUN: | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22u64 \ +// RUN: | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22s64 \ +// RUN: | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64_zfa \ +// RUN: | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cdbec7baf1bc31b59526442c9d4d5f53aac746eb Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:24:08 -0700 Subject: [PATCH 169/933] [libc] fix up the use of angle includes in include/ (#86027) Performed en-masse via: $ grep -rn "#include /#include "ll$1"/' $ grep -rn "#include <__" libc/include -l | \ xargs perl -pi -e 's/#include <__(.*)>/#include "__$1"/' Link: #83463 Link: #83210 --- libc/include/arpa/inet.h.def | 2 +- libc/include/assert.h.def | 2 +- libc/include/ctype.h.def | 2 +- libc/include/dirent.h.def | 2 +- libc/include/errno.h.def | 4 ++-- libc/include/fcntl.h.def | 4 ++-- libc/include/features.h.def | 4 ++-- libc/include/fenv.h.def | 4 ++-- libc/include/float.h.def | 2 +- libc/include/gpu/rpc.h.def | 4 ++-- libc/include/inttypes.h.def | 4 ++-- libc/include/limits.h.def | 2 +- .../llvm-libc-macros/containerof-macro.h | 2 +- .../llvm-libc-macros/sys-queue-macros.h | 4 ++-- libc/include/llvm-libc-types/__mutex_type.h | 2 +- .../llvm-libc-types/cookie_io_functions_t.h | 6 +++--- libc/include/llvm-libc-types/fd_set.h | 2 +- libc/include/llvm-libc-types/mtx_t.h | 2 +- libc/include/llvm-libc-types/once_flag.h | 2 +- libc/include/llvm-libc-types/pthread_attr_t.h | 2 +- .../include/llvm-libc-types/pthread_mutex_t.h | 2 +- libc/include/llvm-libc-types/pthread_once_t.h | 2 +- libc/include/llvm-libc-types/pthread_t.h | 2 +- libc/include/llvm-libc-types/siginfo_t.h | 8 ++++---- libc/include/llvm-libc-types/sigset_t.h | 2 +- libc/include/llvm-libc-types/stack_t.h | 2 +- libc/include/llvm-libc-types/struct_dirent.h | 4 ++-- .../llvm-libc-types/struct_epoll_event.h | 2 +- libc/include/llvm-libc-types/struct_rlimit.h | 2 +- libc/include/llvm-libc-types/struct_rusage.h | 2 +- .../llvm-libc-types/struct_sched_param.h | 6 +++--- .../llvm-libc-types/struct_sigaction.h | 4 ++-- .../include/llvm-libc-types/struct_sockaddr.h | 2 +- .../llvm-libc-types/struct_sockaddr_un.h | 2 +- libc/include/llvm-libc-types/struct_stat.h | 20 +++++++++---------- libc/include/llvm-libc-types/struct_termios.h | 6 +++--- .../include/llvm-libc-types/struct_timespec.h | 2 +- libc/include/llvm-libc-types/struct_timeval.h | 4 ++-- libc/include/llvm-libc-types/thrd_t.h | 2 +- libc/include/math.h.def | 6 +++--- libc/include/pthread.h.def | 2 +- libc/include/sched.h.def | 4 ++-- libc/include/search.h.def | 2 +- libc/include/setjmp.h.def | 2 +- libc/include/signal.h.def | 4 ++-- libc/include/spawn.h.def | 2 +- libc/include/stdbit.h.def | 4 ++-- libc/include/stdckdint.h.def | 4 ++-- libc/include/stdfix.h.def | 4 ++-- libc/include/stdint.h.def | 2 +- libc/include/stdio.h.def | 6 +++--- libc/include/stdlib.h.def | 4 ++-- libc/include/string.h.def | 4 ++-- libc/include/strings.h.def | 2 +- libc/include/sys/auxv.h.def | 4 ++-- libc/include/sys/epoll.h.def | 2 +- libc/include/sys/ioctl.h.def | 4 ++-- libc/include/sys/mman.h.def | 4 ++-- libc/include/sys/prctl.h.def | 2 +- libc/include/sys/queue.h | 2 +- libc/include/sys/random.h.def | 4 ++-- libc/include/sys/resource.h.def | 4 ++-- libc/include/sys/select.h.def | 4 ++-- libc/include/sys/sendfile.h.def | 2 +- libc/include/sys/socket.h.def | 4 ++-- libc/include/sys/stat.h.def | 4 ++-- libc/include/sys/time.h.def | 6 +++--- libc/include/sys/types.h.def | 2 +- libc/include/sys/utsname.h.def | 2 +- libc/include/sys/wait.h.def | 4 ++-- libc/include/termios.h.def | 4 ++-- libc/include/threads.h.def | 2 +- libc/include/time.h.def | 4 ++-- libc/include/uchar.h.def | 2 +- libc/include/unistd.h.def | 6 +++--- libc/include/wchar.h.def | 4 ++-- 76 files changed, 131 insertions(+), 131 deletions(-) diff --git a/libc/include/arpa/inet.h.def b/libc/include/arpa/inet.h.def index fdd5ae3e3f85..6a62b2c7be81 100644 --- a/libc/include/arpa/inet.h.def +++ b/libc/include/arpa/inet.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ARPA_INET_H #define LLVM_LIBC_ARPA_INET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #include diff --git a/libc/include/assert.h.def b/libc/include/assert.h.def index e5d7dfbffdbb..e006133a7654 100644 --- a/libc/include/assert.h.def +++ b/libc/include/assert.h.def @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // This file may be usefully included multiple times to change assert()'s // definition based on NDEBUG. diff --git a/libc/include/ctype.h.def b/libc/include/ctype.h.def index ac52a36bf72f..a9bb786931ea 100644 --- a/libc/include/ctype.h.def +++ b/libc/include/ctype.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_CTYPE_H #define LLVM_LIBC_CTYPE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/dirent.h.def b/libc/include/dirent.h.def index 3de8b1c6713f..6786578fbd06 100644 --- a/libc/include/dirent.h.def +++ b/libc/include/dirent.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_DIRENT_H #define LLVM_LIBC_DIRENT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/errno.h.def b/libc/include/errno.h.def index 90bd8bfecf2f..d7ae90ad4524 100644 --- a/libc/include/errno.h.def +++ b/libc/include/errno.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ERRNO_H #define LLVM_LIBC_ERRNO_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #ifdef __linux__ @@ -40,7 +40,7 @@ #endif // ENOTRECOVERABLE #else // __linux__ -#include +#include "llvm-libc-macros/generic-error-number-macros.h" #endif #if !defined(__AMDGPU__) && !defined(__NVPTX__) diff --git a/libc/include/fcntl.h.def b/libc/include/fcntl.h.def index b11645d18c5b..4f608845ce1e 100644 --- a/libc/include/fcntl.h.def +++ b/libc/include/fcntl.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FCNTL_H #define LLVM_LIBC_FCNTL_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fcntl-macros.h" %%public_api() diff --git a/libc/include/features.h.def b/libc/include/features.h.def index 64205f57acb5..238b88d4b90f 100644 --- a/libc/include/features.h.def +++ b/libc/include/features.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FEATURES_H #define LLVM_LIBC_FEATURES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/features-macros.h" %%public_api() diff --git a/libc/include/fenv.h.def b/libc/include/fenv.h.def index f131a44914ab..c677b2a5930d 100644 --- a/libc/include/fenv.h.def +++ b/libc/include/fenv.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FENV_H #define LLVM_LIBC_FENV_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fenv-macros.h" %%public_api() diff --git a/libc/include/float.h.def b/libc/include/float.h.def index 6d3599d78c69..3bcd7f5e3f98 100644 --- a/libc/include/float.h.def +++ b/libc/include/float.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_FLOAT_H #define LLVM_LIBC_FLOAT_H -#include +#include "llvm-libc-macros/float-macros.h" #endif // LLVM_LIBC_FLOAT_H diff --git a/libc/include/gpu/rpc.h.def b/libc/include/gpu/rpc.h.def index 0438cd65e7be..72acf0c81422 100644 --- a/libc/include/gpu/rpc.h.def +++ b/libc/include/gpu/rpc.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_GPU_RPC_H #define LLVM_LIBC_GPU_RPC_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/rpc_opcodes_t.h" %%public_api() diff --git a/libc/include/inttypes.h.def b/libc/include/inttypes.h.def index a99d4e931f51..5879d2d8e041 100644 --- a/libc/include/inttypes.h.def +++ b/libc/include/inttypes.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_INTTYPES_H #define LLVM_LIBC_INTTYPES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/inttypes-macros.h" #include %%public_api() diff --git a/libc/include/limits.h.def b/libc/include/limits.h.def index de5f3490459e..c37c97c69a84 100644 --- a/libc/include/limits.h.def +++ b/libc/include/limits.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_LIMITS_H #define LLVM_LIBC_LIMITS_H -#include +#include "llvm-libc-macros/limits-macros.h" #endif // LLVM_LIBC_LIMITS_H diff --git a/libc/include/llvm-libc-macros/containerof-macro.h b/libc/include/llvm-libc-macros/containerof-macro.h index 62724abd3b0f..592acd6e3aa9 100644 --- a/libc/include/llvm-libc-macros/containerof-macro.h +++ b/libc/include/llvm-libc-macros/containerof-macro.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H #define LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H -#include +#include "llvm-libc-macros/offsetof-macro.h" #define __containerof(ptr, type, member) \ ({ \ diff --git a/libc/include/llvm-libc-macros/sys-queue-macros.h b/libc/include/llvm-libc-macros/sys-queue-macros.h index fcac265333fc..089b6abaa024 100644 --- a/libc/include/llvm-libc-macros/sys-queue-macros.h +++ b/libc/include/llvm-libc-macros/sys-queue-macros.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H #define LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H -#include -#include +#include "llvm-libc-macros/containerof-macro.h" +#include "llvm-libc-macros/null-macro.h" #ifdef __cplusplus #define QUEUE_TYPEOF(type) type diff --git a/libc/include/llvm-libc-types/__mutex_type.h b/libc/include/llvm-libc-types/__mutex_type.h index d27bf5db8377..3779c78203ed 100644 --- a/libc/include/llvm-libc-types/__mutex_type.h +++ b/libc/include/llvm-libc-types/__mutex_type.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES___MUTEX_TYPE_H #define LLVM_LIBC_TYPES___MUTEX_TYPE_H -#include +#include "llvm-libc-types/__futex_word.h" typedef struct { unsigned char __timed; diff --git a/libc/include/llvm-libc-types/cookie_io_functions_t.h b/libc/include/llvm-libc-types/cookie_io_functions_t.h index f9fa1a2d50ed..a3e7c32a5096 100644 --- a/libc/include/llvm-libc-types/cookie_io_functions_t.h +++ b/libc/include/llvm-libc-types/cookie_io_functions_t.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H #define LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H -#include -#include -#include +#include "llvm-libc-types/off64_t.h" +#include "llvm-libc-types/size_t.h" +#include "llvm-libc-types/ssize_t.h" typedef ssize_t cookie_read_function_t(void *, char *, size_t); typedef ssize_t cookie_write_function_t(void *, const char *, size_t); diff --git a/libc/include/llvm-libc-types/fd_set.h b/libc/include/llvm-libc-types/fd_set.h index 58fc438bbdd2..fd1bde24c90e 100644 --- a/libc/include/llvm-libc-types/fd_set.h +++ b/libc/include/llvm-libc-types/fd_set.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_FD_SET_H #define LLVM_LIBC_TYPES_FD_SET_H -#include // FD_SETSIZE +#include "llvm-libc-macros/sys-select-macros.h" // FD_SETSIZE typedef struct { __FD_SET_WORD_TYPE __set[__FD_SET_ARRAYSIZE]; diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/mtx_t.h index 0f3882c26b6b..ebf79871c935 100644 --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/mtx_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_MTX_T_H #define LLVM_LIBC_TYPES_MTX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type mtx_t; diff --git a/libc/include/llvm-libc-types/once_flag.h b/libc/include/llvm-libc-types/once_flag.h index cb8011284610..f80d35e317e9 100644 --- a/libc/include/llvm-libc-types/once_flag.h +++ b/libc/include/llvm-libc-types/once_flag.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_ONCE_FLAG_H #define LLVM_LIBC_TYPES_ONCE_FLAG_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word once_flag; diff --git a/libc/include/llvm-libc-types/pthread_attr_t.h b/libc/include/llvm-libc-types/pthread_attr_t.h index 66c04de04a99..7512193ef97b 100644 --- a/libc/include/llvm-libc-types/pthread_attr_t.h +++ b/libc/include/llvm-libc-types/pthread_attr_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H #define LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { int __detachstate; diff --git a/libc/include/llvm-libc-types/pthread_mutex_t.h b/libc/include/llvm-libc-types/pthread_mutex_t.h index b1eb21f24fac..cf2194d719f3 100644 --- a/libc/include/llvm-libc-types/pthread_mutex_t.h +++ b/libc/include/llvm-libc-types/pthread_mutex_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H #define LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type pthread_mutex_t; diff --git a/libc/include/llvm-libc-types/pthread_once_t.h b/libc/include/llvm-libc-types/pthread_once_t.h index 3fe78b7ddff6..8ea926f4ee7d 100644 --- a/libc/include/llvm-libc-types/pthread_once_t.h +++ b/libc/include/llvm-libc-types/pthread_once_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H #define LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word pthread_once_t; diff --git a/libc/include/llvm-libc-types/pthread_t.h b/libc/include/llvm-libc-types/pthread_t.h index 72c14e1c2eea..63cc0d7dd74c 100644 --- a/libc/include/llvm-libc-types/pthread_t.h +++ b/libc/include/llvm-libc-types/pthread_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_T_H #define LLVM_LIBC_TYPES_PTHREAD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type pthread_t; diff --git a/libc/include/llvm-libc-types/siginfo_t.h b/libc/include/llvm-libc-types/siginfo_t.h index 935ef4bbcb72..dafe9c1b5f8e 100644 --- a/libc/include/llvm-libc-types/siginfo_t.h +++ b/libc/include/llvm-libc-types/siginfo_t.h @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_TYPES_SIGINFO_T_H #define LLVM_LIBC_TYPES_SIGINFO_T_H -#include -#include -#include -#include +#include "llvm-libc-types/clock_t.h" +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/uid_t.h" +#include "llvm-libc-types/union_sigval.h" #define SI_MAX_SIZE 128 diff --git a/libc/include/llvm-libc-types/sigset_t.h b/libc/include/llvm-libc-types/sigset_t.h index f159c6c6c643..311a92b823ff 100644 --- a/libc/include/llvm-libc-types/sigset_t.h +++ b/libc/include/llvm-libc-types/sigset_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_SIGSET_T_H #define LLVM_LIBC_TYPES_SIGSET_T_H -#include +#include "llvm-libc-macros/signal-macros.h" // This definition can be adjusted/specialized for different targets and // platforms as necessary. This definition works for Linux on most targets. diff --git a/libc/include/llvm-libc-types/stack_t.h b/libc/include/llvm-libc-types/stack_t.h index 5fa4d3a6d3dc..9156425436e9 100644 --- a/libc/include/llvm-libc-types/stack_t.h +++ b/libc/include/llvm-libc-types/stack_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STACK_T_H #define LLVM_LIBC_TYPES_STACK_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { // The order of the fields declared here should match the kernel definition diff --git a/libc/include/llvm-libc-types/struct_dirent.h b/libc/include/llvm-libc-types/struct_dirent.h index 3c5b361c3cbc..0bb71b9f3b84 100644 --- a/libc/include/llvm-libc-types/struct_dirent.h +++ b/libc/include/llvm-libc-types/struct_dirent.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_DIRENT_H #define LLVM_LIBC_TYPES_STRUCT_DIRENT_H -#include -#include +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/off_t.h" struct dirent { ino_t d_ino; diff --git a/libc/include/llvm-libc-types/struct_epoll_event.h b/libc/include/llvm-libc-types/struct_epoll_event.h index 6fc5b410348a..66cf86c1e2a0 100644 --- a/libc/include/llvm-libc-types/struct_epoll_event.h +++ b/libc/include/llvm-libc-types/struct_epoll_event.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H #define LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H -#include +#include "llvm-libc-types/struct_epoll_data.h" typedef struct epoll_event { __UINT32_TYPE__ events; diff --git a/libc/include/llvm-libc-types/struct_rlimit.h b/libc/include/llvm-libc-types/struct_rlimit.h index e093d9f306c9..11e6bee15f9d 100644 --- a/libc/include/llvm-libc-types/struct_rlimit.h +++ b/libc/include/llvm-libc-types/struct_rlimit.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RLIMIT_H #define LLVM_LIBC_TYPES_STRUCT_RLIMIT_H -#include +#include "llvm-libc-types/rlim_t.h" struct rlimit { rlim_t rlim_cur; diff --git a/libc/include/llvm-libc-types/struct_rusage.h b/libc/include/llvm-libc-types/struct_rusage.h index 21ea8b1061c2..ed838d30ede3 100644 --- a/libc/include/llvm-libc-types/struct_rusage.h +++ b/libc/include/llvm-libc-types/struct_rusage.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RUSAGE_H #define LLVM_LIBC_TYPES_STRUCT_RUSAGE_H -#include +#include "llvm-libc-types/struct_timeval.h" struct rusage { struct timeval ru_utime; diff --git a/libc/include/llvm-libc-types/struct_sched_param.h b/libc/include/llvm-libc-types/struct_sched_param.h index 0521a4df652f..86209ac3a181 100644 --- a/libc/include/llvm-libc-types/struct_sched_param.h +++ b/libc/include/llvm-libc-types/struct_sched_param.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H #define LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H -#include -#include -#include +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/time_t.h" struct sched_param { // Process or thread execution scheduling priority. diff --git a/libc/include/llvm-libc-types/struct_sigaction.h b/libc/include/llvm-libc-types/struct_sigaction.h index 54d2995f4ecd..ffce04d0f7e8 100644 --- a/libc/include/llvm-libc-types/struct_sigaction.h +++ b/libc/include/llvm-libc-types/struct_sigaction.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SIGACTION_H #define LLVM_LIBC_TYPES_STRUCT_SIGACTION_H -#include -#include +#include "llvm-libc-types/siginfo_t.h" +#include "llvm-libc-types/sigset_t.h" struct sigaction { union { diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h index 074b1ae50ef0..a98606323c52 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr.h +++ b/libc/include/llvm-libc-types/struct_sockaddr.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H -#include +#include "llvm-libc-types/sa_family_t.h" struct sockaddr { sa_family_t sa_family; diff --git a/libc/include/llvm-libc-types/struct_sockaddr_un.h b/libc/include/llvm-libc-types/struct_sockaddr_un.h index 4332419a5b71..3c0362ce24fb 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr_un.h +++ b/libc/include/llvm-libc-types/struct_sockaddr_un.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H -#include +#include "llvm-libc-types/sa_family_t.h" // This is the sockaddr specialization for AF_UNIX or AF_LOCAL sockets, as // defined by posix. diff --git a/libc/include/llvm-libc-types/struct_stat.h b/libc/include/llvm-libc-types/struct_stat.h index 3539fb5b920e..d8ae9dd6ffdc 100644 --- a/libc/include/llvm-libc-types/struct_stat.h +++ b/libc/include/llvm-libc-types/struct_stat.h @@ -9,16 +9,16 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_STAT_H #define LLVM_LIBC_TYPES_STRUCT_STAT_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "llvm-libc-types/blkcnt_t.h" +#include "llvm-libc-types/blksize_t.h" +#include "llvm-libc-types/dev_t.h" +#include "llvm-libc-types/gid_t.h" +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/mode_t.h" +#include "llvm-libc-types/nlink_t.h" +#include "llvm-libc-types/off_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/uid_t.h" struct stat { dev_t st_dev; diff --git a/libc/include/llvm-libc-types/struct_termios.h b/libc/include/llvm-libc-types/struct_termios.h index 72aefe4f6926..51241192f741 100644 --- a/libc/include/llvm-libc-types/struct_termios.h +++ b/libc/include/llvm-libc-types/struct_termios.h @@ -9,9 +9,9 @@ #ifndef __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ #define __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ -#include -#include -#include +#include "llvm-libc-types/cc_t.h" +#include "llvm-libc-types/speed_t.h" +#include "llvm-libc-types/tcflag_t.h" struct termios { tcflag_t c_iflag; // Input mode flags diff --git a/libc/include/llvm-libc-types/struct_timespec.h b/libc/include/llvm-libc-types/struct_timespec.h index 5d56d9c9468b..4baab07c10f8 100644 --- a/libc/include/llvm-libc-types/struct_timespec.h +++ b/libc/include/llvm-libc-types/struct_timespec.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H #define LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H -#include +#include "llvm-libc-types/time_t.h" struct timespec { time_t tv_sec; /* Seconds. */ diff --git a/libc/include/llvm-libc-types/struct_timeval.h b/libc/include/llvm-libc-types/struct_timeval.h index 6a0b7bbaf825..365b835d345d 100644 --- a/libc/include/llvm-libc-types/struct_timeval.h +++ b/libc/include/llvm-libc-types/struct_timeval.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H #define LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H -#include -#include +#include "llvm-libc-types/suseconds_t.h" +#include "llvm-libc-types/time_t.h" struct timeval { time_t tv_sec; // Seconds diff --git a/libc/include/llvm-libc-types/thrd_t.h b/libc/include/llvm-libc-types/thrd_t.h index 2e0f9a0d75ad..751ea5b9e4c0 100644 --- a/libc/include/llvm-libc-types/thrd_t.h +++ b/libc/include/llvm-libc-types/thrd_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_THRD_T_H #define LLVM_LIBC_TYPES_THRD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type thrd_t; diff --git a/libc/include/math.h.def b/libc/include/math.h.def index 927e2d6697c6..cd2fe76f40bf 100644 --- a/libc/include/math.h.def +++ b/libc/include/math.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_MATH_H #define LLVM_LIBC_MATH_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/math-macros.h" +#include "llvm-libc-types/float128.h" %%public_api() diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def index 391ecd3c124f..abeb839ee83d 100644 --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_PTHREAD_H #define LLVM_LIBC_PTHREAD_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define PTHREAD_STACK_MIN (1 << 14) // 16KB diff --git a/libc/include/sched.h.def b/libc/include/sched.h.def index 3b2d5e330859..493028e8dcc4 100644 --- a/libc/include/sched.h.def +++ b/libc/include/sched.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_SCHED_H #define LLVM_LIBC_SCHED_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/sched-macros.h" %%public_api() diff --git a/libc/include/search.h.def b/libc/include/search.h.def index 3435c1f8ad04..6301ba7b656c 100644 --- a/libc/include/search.h.def +++ b/libc/include/search.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SEARCH_H #define LLVM_LIBC_SEARCH_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include diff --git a/libc/include/setjmp.h.def b/libc/include/setjmp.h.def index 7447be2415bd..670bc1ac0fe2 100644 --- a/libc/include/setjmp.h.def +++ b/libc/include/setjmp.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SETJMP_H #define LLVM_LIBC_SETJMP_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/signal.h.def b/libc/include/signal.h.def index 0e7033452715..50a5f44c7337 100644 --- a/libc/include/signal.h.def +++ b/libc/include/signal.h.def @@ -9,12 +9,12 @@ #ifndef LLVM_LIBC_SIGNAL_H #define LLVM_LIBC_SIGNAL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include -#include +#include "llvm-libc-macros/signal-macros.h" %%public_api() diff --git a/libc/include/spawn.h.def b/libc/include/spawn.h.def index 368ebff17ca1..a8d701585286 100644 --- a/libc/include/spawn.h.def +++ b/libc/include/spawn.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SPAWN_H #define LLVM_LIBC_SPAWN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/stdbit.h.def b/libc/include/stdbit.h.def index c5a77329fbfe..28c147b01e22 100644 --- a/libc/include/stdbit.h.def +++ b/libc/include/stdbit.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDBIT_H #define LLVM_LIBC_STDBIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdbit-macros.h" #endif // LLVM_LIBC_STDBIT_H diff --git a/libc/include/stdckdint.h.def b/libc/include/stdckdint.h.def index c82470911c33..d4a9d829a3c9 100644 --- a/libc/include/stdckdint.h.def +++ b/libc/include/stdckdint.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDCKDINT_H #define LLVM_LIBC_STDCKDINT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdckdint-macros.h" #endif // LLVM_LIBC_STDCKDINT_H diff --git a/libc/include/stdfix.h.def b/libc/include/stdfix.h.def index 368eeb33f2f0..8ac49be45fba 100644 --- a/libc/include/stdfix.h.def +++ b/libc/include/stdfix.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDFIX_H #define LLVM_LIBC_STDFIX_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdfix-macros.h" // From ISO/IEC TR 18037:2008 standard: // https://www.iso.org/standard/51126.html diff --git a/libc/include/stdint.h.def b/libc/include/stdint.h.def index 9e269101acd2..d7660860c91f 100644 --- a/libc/include/stdint.h.def +++ b/libc/include/stdint.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_STDINT_H #define LLVM_LIBC_STDINT_H -#include +#include "llvm-libc-macros/stdint-macros.h" #endif // LLVM_LIBC_STDINT_H diff --git a/libc/include/stdio.h.def b/libc/include/stdio.h.def index a28d009288d5..78d800c83b51 100644 --- a/libc/include/stdio.h.def +++ b/libc/include/stdio.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STDIO_H #define LLVM_LIBC_STDIO_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/stdio-macros.h" #include diff --git a/libc/include/stdlib.h.def b/libc/include/stdlib.h.def index 18df71a49a9b..d523f7a53024 100644 --- a/libc/include/stdlib.h.def +++ b/libc/include/stdlib.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDLIB_H #define LLVM_LIBC_STDLIB_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdlib-macros.h" %%public_api() diff --git a/libc/include/string.h.def b/libc/include/string.h.def index 26e6ef93d314..1bd2687db2be 100644 --- a/libc/include/string.h.def +++ b/libc/include/string.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STRING_H #define LLVM_LIBC_STRING_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/null-macro.h" %%public_api() diff --git a/libc/include/strings.h.def b/libc/include/strings.h.def index f07ca30d5dbd..9b016bf0bc50 100644 --- a/libc/include/strings.h.def +++ b/libc/include/strings.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_STRINGS_H #define LLVM_LIBC_STRINGS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/auxv.h.def b/libc/include/sys/auxv.h.def index 504c2f68cb1e..11ab25bcfe2c 100644 --- a/libc/include/sys/auxv.h.def +++ b/libc/include/sys/auxv.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_AUXV_H #define LLVM_LIBC_SYS_AUXV_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-auxv-macros.h" %%public_api() diff --git a/libc/include/sys/epoll.h.def b/libc/include/sys/epoll.h.def index 490fad91db3c..85f7d9ad6091 100644 --- a/libc/include/sys/epoll.h.def +++ b/libc/include/sys/epoll.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_EPOLL_H #define LLVM_LIBC_SYS_EPOLL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/ioctl.h.def b/libc/include/sys/ioctl.h.def index 90d91cf38291..2f37a1190ac1 100644 --- a/libc/include/sys/ioctl.h.def +++ b/libc/include/sys/ioctl.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_IOCTL_H #define LLVM_LIBC_SYS_IOCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-ioctl-macros.h" %%public_api() diff --git a/libc/include/sys/mman.h.def b/libc/include/sys/mman.h.def index ab9fde1bb920..2e2c2f1997b8 100644 --- a/libc/include/sys/mman.h.def +++ b/libc/include/sys/mman.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_MMAN_H #define LLVM_LIBC_SYS_MMAN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-mman-macros.h" %%public_api() diff --git a/libc/include/sys/prctl.h.def b/libc/include/sys/prctl.h.def index 0a11543d0729..08648c9f4792 100644 --- a/libc/include/sys/prctl.h.def +++ b/libc/include/sys/prctl.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_PRCTL_H #define LLVM_LIBC_SYS_PRCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // Process control is highly platform specific, so the platform usually defines // the macros itself. diff --git a/libc/include/sys/queue.h b/libc/include/sys/queue.h index 1cde35e77a04..cca53c16f0f3 100644 --- a/libc/include/sys/queue.h +++ b/libc/include/sys/queue.h @@ -9,6 +9,6 @@ #ifndef SYS_QUEUE_H #define SYS_QUEUE_H -#include +#include "llvm-libc-macros/sys-queue-macros.h" #endif // SYS_QUEUE_H diff --git a/libc/include/sys/random.h.def b/libc/include/sys/random.h.def index b767f2479fc2..d11431b2755d 100644 --- a/libc/include/sys/random.h.def +++ b/libc/include/sys/random.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RANDOM_H #define LLVM_LIBC_SYS_RANDOM_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-random-macros.h" %%public_api() diff --git a/libc/include/sys/resource.h.def b/libc/include/sys/resource.h.def index 31132d3b2608..365d803cf715 100644 --- a/libc/include/sys/resource.h.def +++ b/libc/include/sys/resource.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RESOURCE_H #define LLVM_LIBC_SYS_RESOURCE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-resource-macros.h" %%public_api() diff --git a/libc/include/sys/select.h.def b/libc/include/sys/select.h.def index 4f3cebaecbb9..529be7158f26 100644 --- a/libc/include/sys/select.h.def +++ b/libc/include/sys/select.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SELECT_H #define LLVM_LIBC_SYS_SELECT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-select-macros.h" %%public_api() diff --git a/libc/include/sys/sendfile.h.def b/libc/include/sys/sendfile.h.def index 947edc28e1ec..d7f21f91f95e 100644 --- a/libc/include/sys/sendfile.h.def +++ b/libc/include/sys/sendfile.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_SENDFILE_H #define LLVM_LIBC_SYS_SENDFILE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/socket.h.def b/libc/include/sys/socket.h.def index 71654c64b988..933ef1512e45 100644 --- a/libc/include/sys/socket.h.def +++ b/libc/include/sys/socket.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SOCKET_H #define LLVM_LIBC_SYS_SOCKET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-socket-macros.h" %%public_api() diff --git a/libc/include/sys/stat.h.def b/libc/include/sys/stat.h.def index ed37d010f497..06a98a4aa029 100644 --- a/libc/include/sys/stat.h.def +++ b/libc/include/sys/stat.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_STAT_H #define LLVM_LIBC_SYS_STAT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-stat-macros.h" %%public_api() diff --git a/libc/include/sys/time.h.def b/libc/include/sys/time.h.def index 9a3bd7bb49f8..5a87139aefc9 100644 --- a/libc/include/sys/time.h.def +++ b/libc/include/sys/time.h.def @@ -9,11 +9,11 @@ #ifndef LLVM_LIBC_SYS_TIME_H #define LLVM_LIBC_SYS_TIME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/struct_timeval.h" -#include +#include "llvm-libc-macros/sys-time-macros.h" %%public_api() diff --git a/libc/include/sys/types.h.def b/libc/include/sys/types.h.def index 689482973fc7..f5c3bb2c928b 100644 --- a/libc/include/sys/types.h.def +++ b/libc/include/sys/types.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_TYPES_H #define LLVM_LIBC_SYS_TYPES_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/utsname.h.def b/libc/include/sys/utsname.h.def index 6d7daeb45f01..08dbbfc06245 100644 --- a/libc/include/sys/utsname.h.def +++ b/libc/include/sys/utsname.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_UTSNAME_H #define LLVM_LIBC_SYS_UTSNAME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/wait.h.def b/libc/include/sys/wait.h.def index b4fcce4d1652..0a76da019fdc 100644 --- a/libc/include/sys/wait.h.def +++ b/libc/include/sys/wait.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_WAIT_H #define LLVM_LIBC_SYS_WAIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-wait-macros.h" %%public_api() diff --git a/libc/include/termios.h.def b/libc/include/termios.h.def index be1cd2bff526..7538944c0985 100644 --- a/libc/include/termios.h.def +++ b/libc/include/termios.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TERMIOS_H #define LLVM_LIBC_TERMIOS_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/termios-macros.h" %%public_api() diff --git a/libc/include/threads.h.def b/libc/include/threads.h.def index 93541b8d3bac..b114bea0ace3 100644 --- a/libc/include/threads.h.def +++ b/libc/include/threads.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_THREADS_H #define LLVM_LIBC_THREADS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/time.h.def b/libc/include/time.h.def index d8988329a372..2355e8822fad 100644 --- a/libc/include/time.h.def +++ b/libc/include/time.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TIME_H #define LLVM_LIBC_TIME_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/time-macros.h" %%public_api() diff --git a/libc/include/uchar.h.def b/libc/include/uchar.h.def index 7e62d43e9cc4..31b7fcb73ded 100644 --- a/libc/include/uchar.h.def +++ b/libc/include/uchar.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_UCHAR_H #define LLVM_LIBC_UCHAR_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/unistd.h.def b/libc/include/unistd.h.def index fa10af653fae..6b9137e14623 100644 --- a/libc/include/unistd.h.def +++ b/libc/include/unistd.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_UNISTD_H #define LLVM_LIBC_UNISTD_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/unistd-macros.h" %%public_api() diff --git a/libc/include/wchar.h.def b/libc/include/wchar.h.def index ac72f80aa083..4c25de700d60 100644 --- a/libc/include/wchar.h.def +++ b/libc/include/wchar.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_WCHAR_H #define LLVM_LIBC_WCHAR_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/wchar-macros.h" %%public_api() -- GitLab From 4318f7e5301fb737a7abaacb3b43b6a9289055f3 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:28:03 -0700 Subject: [PATCH 170/933] [libc][stdlib] initial support for __cxa_finalize (#85865) I'm trying to break up the pieces of supporting __cxa_finalize into smaller commits. Provide this symbol first, and make use of it from exit. Next will be to store __dso_handle, then finally to only run callbacks that were registered from a specific dso. Link: #85651 Link: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#dso-dtor --- libc/src/stdlib/atexit.cpp | 31 +++++++++++++++++++------------ libc/src/stdlib/exit.cpp | 8 +++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp index 741ea4f25103..fa072b2fdf8d 100644 --- a/libc/src/stdlib/atexit.cpp +++ b/libc/src/stdlib/atexit.cpp @@ -55,14 +55,10 @@ void stdc_at_exit_func(void *payload) { reinterpret_cast(payload)(); } -} // namespace - -namespace internal { - void call_exit_callbacks() { handler_list_mtx.lock(); while (!exit_callbacks.empty()) { - auto unit = exit_callbacks.back(); + AtExitUnit &unit = exit_callbacks.back(); exit_callbacks.pop_back(); handler_list_mtx.unlock(); unit.callback(unit.payload); @@ -71,20 +67,31 @@ void call_exit_callbacks() { ExitCallbackList::destroy(&exit_callbacks); } -} // namespace internal - -static int add_atexit_unit(const AtExitUnit &unit) { +int add_atexit_unit(const AtExitUnit &unit) { MutexLock lock(&handler_list_mtx); - if (!exit_callbacks.push_back(unit)) - return -1; - return 0; + if (exit_callbacks.push_back(unit)) + return 0; + return -1; } +} // namespace + +extern "C" { + // TODO: Handle the last dso handle argument. -extern "C" int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { +int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { return add_atexit_unit({callback, payload}); } +// TODO: Handle the dso handle argument. call_exit_callbacks should only invoke +// the callbacks from this DSO. Requires adding support for __dso_handle. +void __cxa_finalize(void *dso) { + if (!dso) + call_exit_callbacks(); +} + +} // extern "C" + LLVM_LIBC_FUNCTION(int, atexit, (StdCAtExitCallback * callback)) { return add_atexit_unit( {&stdc_at_exit_func, reinterpret_cast(callback)}); diff --git a/libc/src/stdlib/exit.cpp b/libc/src/stdlib/exit.cpp index cc5ae6648d11..e754b34e4698 100644 --- a/libc/src/stdlib/exit.cpp +++ b/libc/src/stdlib/exit.cpp @@ -10,14 +10,12 @@ #include "src/__support/OSUtil/quick_exit.h" #include "src/__support/common.h" -namespace LIBC_NAMESPACE { +extern "C" void __cxa_finalize(void *); -namespace internal { -void call_exit_callbacks(); -} +namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(void, exit, (int status)) { - internal::call_exit_callbacks(); + __cxa_finalize(nullptr); quick_exit(status); __builtin_unreachable(); } -- GitLab From 6e28ecd79995a72a8dbde8f16a1afc18309442a1 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 16:23:19 +0100 Subject: [PATCH 171/933] [Object][ELF] Ensure offset to locate dyn section does not go past size Validate `p_offset` in `dynamicEntries` before computing the entry offset. Fixes: https://github.com/llvm/llvm-project/issues/85568. --- llvm/lib/Object/ELF.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 55dd0c8e06c0..0ac4e7a57759 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -560,7 +560,11 @@ Expected ELFFile::dynamicEntries() const { for (const Elf_Phdr &Phdr : *ProgramHeadersOrError) { if (Phdr.p_type == ELF::PT_DYNAMIC) { - Dyn = ArrayRef(reinterpret_cast(base() + Phdr.p_offset), + const uint8_t *DynOffset = base() + Phdr.p_offset; + if (DynOffset > end()) + return createError( + "dynamic section offset past file size: corrupted ELF"); + Dyn = ArrayRef(reinterpret_cast(DynOffset), Phdr.p_filesz / sizeof(Elf_Dyn)); break; } -- GitLab From f66d631bf8dc0fe33c6ba88c3dc7f00ac5946065 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:38:40 -0700 Subject: [PATCH 172/933] Revert "[BOLT] Add BB index to BAT (#86044)" This reverts commit 3b3de48fd84b8269d5f45ee0a9dc6b7448368424. --- bolt/docs/BAT.md | 11 +++-- .../bolt/Profile/BoltAddressTranslation.h | 7 +--- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 +++++------------ .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 + lld/MachO/Driver.cpp | 42 ++----------------- lld/MachO/InputSection.cpp | 38 +++++++++++++++++ lld/MachO/InputSection.h | 3 ++ lld/MachO/ObjC.cpp | 16 +++---- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 77 insertions(+), 92 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..186b0e5ea89d 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,12 +90,11 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | Branch/BB | -| ------ | ------| ----------- | ------ | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | -| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | -| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | +| Entry | Encoding | Description | +| ------ | ------| ----------- | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | +| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..1f53f6d344ad 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,10 +122,6 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -158,8 +154,7 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; + using BBHashMap = std::unordered_map; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..1d61a1b735b4 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,8 +45,6 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -219,7 +217,6 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; - size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -229,15 +226,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + // Basic block hash + size_t BBHash = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - // Basic block index in the input binary - encodeULEB128(BBIndex - PrevBBIndex, OS); - PrevBBIndex = BBIndex; - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, - InOffset >> 1, BBHash, BBIndex)); + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, + InOffset >> 1, BBHash)); } } } @@ -323,7 +316,6 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; - size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -338,25 +330,19 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; - size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); - BBIndexDelta = DE.getULEB128(&Offset, &Err); - BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (!IsBranchEntry) { - dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, - getULEB128Size(BBIndexDelta)); - } + if (BBHash) + dbgs() << formatv(" {0:x}", BBHash); dbgs() << '\n'; }); } @@ -508,19 +494,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), + BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..25ff4e7fbfcc 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 5c1db89e3c6b..4277b4e0d0fe 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 920 +# CHECK: BOLT-INFO: BAT section size (bytes): 816 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bc9cc8ce6cf5..86a287db72a4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { - // NVPTX/AMDGPU does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { + // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index f23cb71ebfb0..5618b2cba72e 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,4 +2,5 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all +// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 36248925d65a..919a14b8bcf0 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - inputSections.push_back(isec); + addInputSection(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,53 +1220,18 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); - int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - ConcatOutputSection *osec = nullptr; - for (const Subsection &subsection : section->subsections) { - if (auto *isec = dyn_cast(subsection.isec)) { - if (isec->isCoalescedWeak()) - continue; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - continue; - } - isec->outSecOff = inputOrder++; - if (!osec) - osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - } + for (const Subsection &subsection : section->subsections) + addInputSection(subsection.isec); } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } - assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1422,6 +1387,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); + inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 8f5affb1dc21..22930d52dd1d 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,6 +37,44 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; +int macho::inputSectionsOrder = 0; + +// Call this function to add a new InputSection and have it routed to the +// appropriate container. Depending on its type and current config, it will +// either be added to 'inputSections' vector or to a synthetic section. +void lld::macho::addInputSection(InputSection *inputSection) { + if (auto *isec = dyn_cast(inputSection)) { + if (isec->isCoalescedWeak()) + return; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + return; + } + isec->outSecOff = inputSectionsOrder++; + auto *osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(inputSection)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputSectionsOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSectionsOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = dyn_cast(inputSection)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSectionsOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + + assert(inputSectionsOrder <= UnspecifiedInputOrder); +} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index b25f0638f4c6..694bdf734907 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,6 +302,8 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; +// This is used as a counter for specyfing input order for input sections +extern int inputSectionsOrder; namespace section_names { @@ -369,6 +371,7 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names +void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 40df2243b26f..5902b82d30f5 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - allInputSections.push_back(newCatList); + addInputSection(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - allInputSections.push_back(newBodySec); + addInputSection(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,9 +1221,11 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - auto &data = newSectionData(len + 1); + uint32_t bufSize = len + 1; + auto &data = newSectionData(bufSize); char *strData = reinterpret_cast(data.data()); - strncpy(strData, str, len); + // Copy the string chars and null-terminator + memcpy(strData, str, bufSize); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 7ee3261ce307..1b3694528de1 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - inputSections.push_back(in.imageLoaderCache); + addInputSection(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - inputSections.push_back(objcSelref); + addInputSection(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b0e23639c5b19030bee2b307173802914f64aad6 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Tue, 20 Feb 2024 16:35:08 -0800 Subject: [PATCH 173/933] [BOLT] Add BB index to BAT Add input basic block index to BAT metadata. This addresses the case where some basic blocks are eliminated, and output index is not equal to the input block index. These indices are used in non-stale-matching mode. Increases BAT section size to: - large binary: 39521512 bytes (1.02x original), - medium binary: 3799988 bytes (0.64x), - small binary: 920 bytes (0.64x). Test Plan: Updated bolt-address-translation{,-yaml}.test Pull Request: https://github.com/llvm/llvm-project/pull/86044 --- bolt/docs/BAT.md | 11 +++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions -- GitLab From cb300c33059c1d14f72392ce5dffcf050ad7567d Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 16:44:06 +0100 Subject: [PATCH 174/933] [MLIR][LLVM][SROA] Fix pointer escape through stores bug (#86291) This commit resolves a SROA bug caused by not properly checking if a llvm store operation writes the pointer to memory or not. Now, we do no longer consider stores that use a slot pointer as a value to store as fixable. --- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 4 ++++ mlir/test/Dialect/LLVMIR/sroa.mlir | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 0ef1d105aca6..f171bf7cc4be 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -251,6 +251,10 @@ bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, if (getVolatile_()) return false; + // Storing the pointer to memory cannot be dealt with. + if (getValue() == slot.ptr) + return false; + // A store always accesses the first element of the destructured slot. auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); Type subslotType = getTypeAtIndex(slot, index); diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index ca49b1298b0e..3f4d17c6a43f 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -305,3 +305,16 @@ llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr llvm.return } + +// ----- + +// CHECK-LABEL: llvm.func @store_to_memory +// CHECK-SAME: %[[ARG:.*]]: !llvm.ptr +llvm.func @store_to_memory(%arg: !llvm.ptr) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct< + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ALLOCA]], %[[ARG]] + llvm.store %1, %arg : !llvm.ptr, !llvm.ptr + llvm.return +} -- GitLab From ceba3a38e8f7b378ad20641832d568460892af1d Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:46:48 -0700 Subject: [PATCH 175/933] [BOLT] Add number of basic blocks to BAT YAML profile reader checks the number of basic blocks in regular, no-stale-matching mode. Add it to BAT. This increases the size of BAT section to: - large binary: 39583080 bytes (1.02x of the original), - medium binary: 3816492 bytes (0.64x), - small binary: 920 bytes (0.64x, no change due to alignment). Test Plan: Updated bolt-address-translation-yaml.test Reviewers: rafaelauler, ayermolo, maksfb, dcci Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86045 --- bolt/docs/BAT.md | 1 + bolt/include/bolt/Profile/BoltAddressTranslation.h | 3 +++ bolt/lib/Profile/BoltAddressTranslation.cpp | 10 ++++++++++ bolt/test/X86/bolt-address-translation-yaml.test | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..2279a070263e 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -79,6 +79,7 @@ Hot indices are delta encoded, implicitly starting at zero. | `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table | | `FuncHash` | 8b | Hot functions only: function hash for input function | +| `NumBlocks` | ULEB128 | Hot functions only: number of basic blocks in the original function | | `NumEntries` | ULEB128 | Number of address translation entries for a function | | `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function | | `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..d583ce0b76a2 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -162,6 +162,9 @@ private: using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; + /// Map a function to its basic blocks count + std::unordered_map NumBasicBlocksMap; + /// Links outlined cold bocks to their original function std::map ColdPartSource; diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..31886f4c8025 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -196,6 +196,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Function hash LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); OS.write(reinterpret_cast(&FuncHashPair.first), 8); + // Number of basic blocks + size_t NumBasicBlocks = FuncHashPair.second.size(); + LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); + encodeULEB128(NumBasicBlocks, OS); } encodeULEB128(NumEntries, OS); // For hot fragments only: encode the number of equal offsets @@ -293,6 +297,12 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, const size_t FuncHash = DE.getU64(&Offset, &Err); FuncHashes[Address].first = FuncHash; LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); + // Number of basic blocks + const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); + NumBasicBlocksMap.emplace(Address, NumBasicBlocks); + LLVM_DEBUG(dbgs() << formatv("{0:x}: #bbs {1}, {2} bytes\n", Address, + NumBasicBlocks, + getULEB128Size(NumBasicBlocks))); } const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); // Equal offsets, hot fragments only. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..ee54a90a9f2a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 380 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries -- GitLab From 2091c74796b1dac68e622284c63a870b88b7554f Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 15:47:40 +0000 Subject: [PATCH 176/933] [RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657) Follow on from #84915 which adds the DbgRecord function variants. Update the LLVMDIBuilderInsert... functions to insert DbgRecords instead of debug intrinsics. LLVMDIBuilderInsertDeclareBefore LLVMDIBuilderInsertDeclareAtEnd LLVMDIBuilderInsertDbgValueBefore LLVMDIBuilderInsertDbgValueAtEnd Calling these functions will now cause an assertion if the module is in the wrong debug info format. They should only be used when the module is in "new debug format". Use LLVMIsNewDbgInfoFormat to query and LLVMSetIsNewDbgInfoFormat to change the debug info format of a module. Please see https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-change (RemoveDIsDebugInfo.md) for more info. --- llvm/docs/RemoveDIsDebugInfo.md | 11 ++- llvm/include/llvm-c/DebugInfo.h | 60 +++++++++----- llvm/lib/IR/DebugInfo.cpp | 121 +++++++++++++++++++---------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 137 insertions(+), 68 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index a2f1e173d9d9..9e50a2a604aa 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,15 +40,22 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. + +Existing functions (behaviour change) +------------------------------------- +LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareAtEnd # Same as above. +LLVMDIBuilderInsertDbgValueBefore # Same as above. +LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index b23ff63c862f..dab1d697761b 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,7 +1249,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new llvm.dbg.declare intrinsic call before the given instruction. + * Insert a new Declare DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1257,13 +1262,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1279,7 +1284,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1295,9 +1300,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Declare DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1305,12 +1315,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1328,7 +1338,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1346,7 +1356,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new llvm.dbg.value intrinsic call before the given instruction. + * Insert a new Value DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1354,13 +1369,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1376,7 +1391,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1392,9 +1407,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.value intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Value DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1402,12 +1422,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1425,7 +1445,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 09bce9df1f33..4206162d1768 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, - Expr, DL, Instr); + return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, + DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,27 +1679,38 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return wrap( - unwrap(Builder) - ->insertDeclare(unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, - Expr, DL, Block); + return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, + DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1707,26 +1718,36 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDeclare(unwrap(Storage), - unwrap(VarInfo), - unwrap(Expr), - unwrap(DL), unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1734,26 +1755,36 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1761,19 +1792,29 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 78ccaf12a380..9b5c37b05d90 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,12 +136,13 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); + if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -149,11 +150,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -161,11 +162,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 3054d0dae7a813c493d2bb8e969aa2321145a83b Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Fri, 22 Mar 2024 08:51:53 -0700 Subject: [PATCH 177/933] AMDGPU: Rename and add bf16 support for global_load_tr builtins (#86202) Make the name of a clang builtin as close to the mnemonic instruction name as possible. The data type suffix may not be enough to tell what instruction the builtin is going to produce. This patch also add the bf16 support for global_load_tr_b128 builtins. --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 16 ++++---- clang/lib/CodeGen/CGBuiltin.cpp | 34 +++++++++++------ ...uiltins-amdgcn-global-load-tr-gfx11-err.cl | 25 ++++++------ ...ins-amdgcn-global-load-tr-gfx12-w32-err.cl | 11 +++--- ...ins-amdgcn-global-load-tr-gfx12-w64-err.cl | 11 +++--- .../builtins-amdgcn-global-load-tr-w32.cl | 38 +++++++++---------- .../builtins-amdgcn-global-load-tr-w64.cl | 38 +++++++++---------- 7 files changed, 94 insertions(+), 79 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 61ec8b79bf05..4153b316c22b 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -432,13 +432,15 @@ TARGET_BUILTIN(__builtin_amdgcn_s_wakeup_barrier, "vi", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "b", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") - -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8bf16, "V8yV8y*1", "nc", "gfx12-insts,wavefrontsize32") + +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64") //===----------------------------------------------------------------------===// // WMMA builtins. diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index e14e89088282..2eaceeba6177 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -18531,35 +18531,45 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy}); return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1}); } - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: { + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: { llvm::Type *ArgTy; switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: ArgTy = llvm::Type::getInt32Ty(getLLVMContext()); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt32Ty(getLLVMContext()), 2); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 4); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 8); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 8); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 8); break; diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl index f7afb7cb97ed..4363769b8645 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl @@ -6,21 +6,22 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); - - -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, - global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr, + global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} - - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} + + int out_5 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_6 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_7 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o8 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl index 04ac0a66db7c..208f92fc5d44 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl @@ -5,11 +5,12 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + int out_1 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_2 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_3 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl index 113b54b853a9..199146a9715d 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl @@ -6,11 +6,12 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl index b5fcad68a470..0035b16b902b 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl @@ -5,44 +5,44 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -// Wave32 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v2i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_v2i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <2 x i32> [[TMP0]] // -v2i test_amdgcn_global_load_tr_v2i32(global v2i* inptr) +v2i test_amdgcn_global_load_tr_b64_v2i32(global v2i* inptr) { - return __builtin_amdgcn_global_load_tr_v2i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_v2i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x i16> [[TMP0]] // -v8s test_amdgcn_global_load_tr_v8i16(global v8s* inptr) +v8s test_amdgcn_global_load_tr_b128_v8i16(global v8s* inptr) { - return __builtin_amdgcn_global_load_tr_v8i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x half> [[TMP0]] // -v8h test_amdgcn_global_load_tr_v8f16(global v8h* inptr) +v8h test_amdgcn_global_load_tr_b128_v8f16(global v8h* inptr) { - return __builtin_amdgcn_global_load_tr_v8f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <8 x bfloat> [[TMP0]] +// +v8bf16 test_amdgcn_global_load_tr_b128_v8bf16(global v8bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v8bf16(inptr); +} diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl index 9c48ac071b4d..6c025bb5a55a 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl @@ -4,44 +4,44 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -// Wave64 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret i32 [[TMP0]] // -int test_amdgcn_global_load_tr_i32(global int* inptr) +int test_amdgcn_global_load_tr_b64_i32(global int* inptr) { - return __builtin_amdgcn_global_load_tr_i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x i16> [[TMP0]] // -v4s test_amdgcn_global_load_tr_v4i16(global v4s* inptr) +v4s test_amdgcn_global_load_tr_b128_v4i16(global v4s* inptr) { - return __builtin_amdgcn_global_load_tr_v4i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x half> [[TMP0]] // -v4h test_amdgcn_global_load_tr_v4f16(global v4h* inptr) +v4h test_amdgcn_global_load_tr_b128_v4f16(global v4h* inptr) { - return __builtin_amdgcn_global_load_tr_v4f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <4 x bfloat> [[TMP0]] +// +v4bf16 test_amdgcn_global_load_tr_b128_v4bf16(global v4bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v4bf16(inptr); +} -- GitLab From 631e54aa1a0b7a79d0dec8dce7ec0f5e506acf6c Mon Sep 17 00:00:00 2001 From: "long.chen" Date: Fri, 22 Mar 2024 23:52:47 +0800 Subject: [PATCH 178/933] [mlir][arith] fix wrong floordivsi fold (#83248) Fixs https://github.com/llvm/llvm-project/issues/83079 --- mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 36 ++------ .../Dialect/Arith/Transforms/ExpandOps.cpp | 58 ++++++------- mlir/test/Dialect/Arith/expand-ops.mlir | 84 ++++++++----------- .../Standard/CPU/test-ceil-floor-pos-neg.mlir | 30 +++++++ mlir/test/Transforms/canonicalize.mlir | 9 ++ 5 files changed, 102 insertions(+), 115 deletions(-) diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp index 9f64a07f31e3..2f32d9a26e77 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp +++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp @@ -689,43 +689,17 @@ OpFoldResult arith::FloorDivSIOp::fold(FoldAdaptor adaptor) { return getLhs(); // Don't fold if it would overflow or if it requires a division by zero. - bool overflowOrDiv0 = false; + bool overflowOrDiv = false; auto result = constFoldBinaryOp( adaptor.getOperands(), [&](APInt a, const APInt &b) { - if (overflowOrDiv0 || !b) { - overflowOrDiv0 = true; + if (b.isZero()) { + overflowOrDiv = true; return a; } - if (!a) - return a; - // After this point we know that neither a or b are zero. - unsigned bits = a.getBitWidth(); - APInt zero = APInt::getZero(bits); - bool aGtZero = a.sgt(zero); - bool bGtZero = b.sgt(zero); - if (aGtZero && bGtZero) { - // Both positive, return a / b. - return a.sdiv_ov(b, overflowOrDiv0); - } - if (!aGtZero && !bGtZero) { - // Both negative, return -a / -b. - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - return posA.sdiv_ov(posB, overflowOrDiv0); - } - if (!aGtZero && bGtZero) { - // A is negative, b is positive, return - ceil(-a, b). - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(posA, b, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); - } - // A is positive, b is negative, return - ceil(a, -b). - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(a, posB, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); + return a.sfloordiv_ov(b, overflowOrDiv); }); - return overflowOrDiv0 ? Attribute() : result; + return overflowOrDiv ? Attribute() : result; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp index 7f246daf99ff..71e14a153cfd 100644 --- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp @@ -110,9 +110,13 @@ struct CeilDivSIOpConverter : public OpRewritePattern { } }; -/// Expands FloorDivSIOp (n, m) into -/// 1) x = (m<0) ? 1 : -1 -/// 2) return (n*m<0) ? - ((-n+x) / m) -1 : n / m +/// Expands FloorDivSIOp (x, y) into +/// z = x / y +/// if (z * y != x && (x < 0) != (y < 0)) { +/// return z - 1; +/// } else { +/// return z; +/// } struct FloorDivSIOpConverter : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(arith::FloorDivSIOp op, @@ -121,41 +125,29 @@ struct FloorDivSIOpConverter : public OpRewritePattern { Type type = op.getType(); Value a = op.getLhs(); Value b = op.getRhs(); - Value plusOne = createConst(loc, type, 1, rewriter); + + Value quotient = rewriter.create(loc, a, b); + Value product = rewriter.create(loc, quotient, b); + Value notEqualDivisor = rewriter.create( + loc, arith::CmpIPredicate::ne, a, product); Value zero = createConst(loc, type, 0, rewriter); - Value minusOne = createConst(loc, type, -1, rewriter); - // Compute x = (b<0) ? 1 : -1. - Value compare = - rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value x = rewriter.create(loc, compare, plusOne, minusOne); - // Compute negative res: -1 - ((x-a)/b). - Value xMinusA = rewriter.create(loc, x, a); - Value xMinusADivB = rewriter.create(loc, xMinusA, b); - Value negRes = rewriter.create(loc, minusOne, xMinusADivB); - // Compute positive res: a/b. - Value posRes = rewriter.create(loc, a, b); - // Result is (a*b<0) ? negative result : positive result. - // Note, we want to avoid using a*b because of possible overflow. - // The case that matters are a>0, a==0, a<0, b>0 and b<0. We do - // not particuliarly care if a*b<0 is true or false when b is zero - // as this will result in an illegal divide. So `a*b<0` can be reformulated - // as `(a>0 && b<0) || (a>0 && b<0)' or `(a>0 && b<0) || (a>0 && b<=0)'. - // We pick the first expression here. + Value aNeg = rewriter.create(loc, arith::CmpIPredicate::slt, a, zero); - Value aPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, a, zero); Value bNeg = rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value bPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, b, zero); - Value firstTerm = rewriter.create(loc, aNeg, bPos); - Value secondTerm = rewriter.create(loc, aPos, bNeg); - Value compareRes = - rewriter.create(loc, firstTerm, secondTerm); - // Perform substitution and return success. - rewriter.replaceOpWithNewOp(op, compareRes, negRes, - posRes); + + Value signOpposite = rewriter.create( + loc, arith::CmpIPredicate::ne, aNeg, bNeg); + Value cond = + rewriter.create(loc, notEqualDivisor, signOpposite); + + Value minusOne = createConst(loc, type, -1, rewriter); + Value quotientMinusOne = + rewriter.create(loc, quotient, minusOne); + + rewriter.replaceOpWithNewOp(op, cond, quotientMinusOne, + quotient); return success(); } }; diff --git a/mlir/test/Dialect/Arith/expand-ops.mlir b/mlir/test/Dialect/Arith/expand-ops.mlir index 91f652e5a270..6bed93e4c969 100644 --- a/mlir/test/Dialect/Arith/expand-ops.mlir +++ b/mlir/test/Dialect/Arith/expand-ops.mlir @@ -66,23 +66,17 @@ func.func @ceildivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { %res = arith.floordivsi %arg0, %arg1 : i32 return %res : i32 -// CHECK: [[ONE:%.+]] = arith.constant 1 : i32 -// CHECK: [[ZERO:%.+]] = arith.constant 0 : i32 -// CHECK: [[MIN1:%.+]] = arith.constant -1 : i32 -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : i32 -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : i32 -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : i32 -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : i32 -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : i32 -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : i32 +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : i32 +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : i32 +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : i32 +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : i32 +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : i32 +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : i32 +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant -1 : i32 +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : i32 +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : i32 } // ----- @@ -93,23 +87,17 @@ func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { %res = arith.floordivsi %arg0, %arg1 : index return %res : index -// CHECK: [[ONE:%.+]] = arith.constant 1 : index -// CHECK: [[ZERO:%.+]] = arith.constant 0 : index -// CHECK: [[MIN1:%.+]] = arith.constant -1 : index -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : index -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : index -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : index -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : index -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : index -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : index -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : index -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : index -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : index +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : index +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : index +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : index +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : index +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : index +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : index +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK: %[[NEG_ONE:.*]] = arith.constant -1 : index +// CHECK-DAG: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : index +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : index } // ----- @@ -121,23 +109,17 @@ func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi_vec(%arg0: vector<4xi32>, %arg1: vector<4xi32>) -> (vector<4xi32>) { %res = arith.floordivsi %arg0, %arg1 : vector<4xi32> return %res : vector<4xi32> -// CHECK: %[[VAL_2:.*]] = arith.constant dense<1> : vector<4xi32> -// CHECK: %[[VAL_3:.*]] = arith.constant dense<0> : vector<4xi32> -// CHECK: %[[VAL_4:.*]] = arith.constant dense<-1> : vector<4xi32> -// CHECK: %[[VAL_5:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_6:.*]] = arith.select %[[VAL_5]], %[[VAL_2]], %[[VAL_4]] : vector<4xi1>, vector<4xi32> -// CHECK: %[[VAL_7:.*]] = arith.subi %[[VAL_6]], %[[VAL_0]] : vector<4xi32> -// CHECK: %[[VAL_8:.*]] = arith.divsi %[[VAL_7]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_9:.*]] = arith.subi %[[VAL_4]], %[[VAL_8]] : vector<4xi32> -// CHECK: %[[VAL_10:.*]] = arith.divsi %[[VAL_0]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_13:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_15:.*]] = arith.andi %[[VAL_11]], %[[VAL_14]] : vector<4xi1> -// CHECK: %[[VAL_16:.*]] = arith.andi %[[VAL_12]], %[[VAL_13]] : vector<4xi1> -// CHECK: %[[VAL_17:.*]] = arith.ori %[[VAL_15]], %[[VAL_16]] : vector<4xi1> -// CHECK: %[[VAL_18:.*]] = arith.select %[[VAL_17]], %[[VAL_9]], %[[VAL_10]] : vector<4xi1>, vector<4xi32> +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : vector<4xi32> +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : vector<4xi32> +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : vector<4xi32> +// CHECK-DAG: %[[ZERO:.*]] = arith.constant dense<0> : vector<4xi32> +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : vector<4xi32> +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : vector<4xi32> +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : vector<4xi1> +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : vector<4xi1> +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant dense<-1> : vector<4xi32> +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : vector<4xi32> +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : vector<4xi1>, vector<4xi32> } // ----- diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir index 39fbb67512c6..a7013eacc984 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir @@ -2,6 +2,10 @@ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_c_runner_utils | \ // RUN: FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-cpu-runner -e main -entry-point-result=void \ +// RUN: -shared-libs=%mlir_c_runner_utils | \ +// RUN: FileCheck %s --check-prefix=SCHECK func.func @transfer_read_2d(%A : memref<40xi32>, %base1: index) { %i42 = arith.constant -42: i32 @@ -101,3 +105,29 @@ func.func @entry() { // CHECK:( 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2 ) // CHECK:( 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4 ) // CHECK:( 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + +// ----- + +func.func @non_inline_function() -> (i64, i64, i64, i64, i64, i64) { + %MIN_INT_MINUS_ONE = arith.constant -9223372036854775807 : i64 + %NEG_ONE = arith.constant -1 : i64 + %MIN_INT = arith.constant -9223372036854775808 : i64 + %ONE = arith.constant 1 : i64 + %MAX_INT = arith.constant 9223372036854775807 : i64 + return %MIN_INT_MINUS_ONE, %NEG_ONE, %MIN_INT, %ONE, %MAX_INT, %NEG_ONE : i64, i64, i64, i64, i64, i64 +} + +func.func @main() { + %0:6 = call @non_inline_function() : () -> (i64, i64, i64, i64, i64, i64) + %1 = arith.floordivsi %0#0, %0#1 : i64 + %2 = arith.floordivsi %0#2, %0#3 : i64 + %3 = arith.floordivsi %0#4, %0#5 : i64 + vector.print %1 : i64 + vector.print %2 : i64 + vector.print %3 : i64 + return +} + +// SCHECK: 9223372036854775807 +// SCHECK: -9223372036854775808 +// SCHECK: -9223372036854775807 diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir index 2cf86b50d432..d2c2c12d3238 100644 --- a/mlir/test/Transforms/canonicalize.mlir +++ b/mlir/test/Transforms/canonicalize.mlir @@ -989,6 +989,15 @@ func.func @tensor_arith.floordivsi_by_one(%arg0: tensor<4x5xi32>) -> tensor<4x5x return %res : tensor<4x5xi32> } +// CHECK-LABEL: func @arith.floordivsi_by_one_overflow +func.func @arith.floordivsi_by_one_overflow() -> i64 { + %neg_one = arith.constant -1 : i64 + %min_int = arith.constant -9223372036854775808 : i64 + // CHECK: arith.floordivsi + %poision = arith.floordivsi %min_int, %neg_one : i64 + return %poision : i64 +} + // ----- // CHECK-LABEL: func @arith.ceildivsi_by_one -- GitLab From 01b1b0c1f728e2c2639edc654424f50830295989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 15:53:04 +0000 Subject: [PATCH 179/933] [mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225) Follow-up for https://github.com/llvm/llvm-project/pull/81625 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir new file mode 100644 index 000000000000..57d69383c2de --- /dev/null +++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir @@ -0,0 +1,60 @@ +// DEFINE: %{compile} = mlir-opt %s \ +// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ +// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ +// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t +// DEFINE: %{entry_point} = conv +// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ +// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils + +// RUN: %{compile} | %{run} | FileCheck %s + +func.func @conv() { + // Define input/output tensors + %input_init = tensor.empty() : tensor<1x8x6xi32> + %output_init = tensor.empty() : tensor<1x7x6xi32> + + %five = arith.constant 5 : i32 + %zero = arith.constant 0 : i32 + %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> + %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> + + // Define the filter tensor + %filter = arith.constant dense<[ + [ 1, 2, 3, 4, 5, 6], + [ 11, 12, 13, 14, 15, 16] + ]> : tensor<2x6xi32> + + // static sizes -> dynamic sizes + %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> + %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> + %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> + + // Run the convolution + %res = linalg.depthwise_conv_1d_nwc_wc + ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) + outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> + + // Print the results + // CHECK: SVE: START OF TEST OUTPUT + vector.print str "SVE: START OF TEST OUTPUT\n" + + // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = + // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] + %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> + call @printMemrefI32(%xf) : (tensor<*xi32>) -> () + + // CHECK-NEXT: SVE: END OF TEST OUTPUT + vector.print str "SVE: END OF TEST OUTPUT\n" + + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op + transform.yield + } +} + +func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:04:50 -0700 Subject: [PATCH 180/933] [bazel] Update to 7.x (#86297) --- utils/bazel/.bazelrc | 5 +++++ utils/bazel/.bazelversion | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index c06e9b341626..46894decc7ad 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,6 +6,11 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### + +# Flip off to disable MODULE.bazel until we're ready. +# https://github.com/llvm/llvm-project/issues/55924 +common --enable_bzlmod=false + # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 5e3254243a3b..21c8c7b46b89 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -6.1.2 +7.1.1 -- GitLab From 26857582e5ee7980a71133ef8f8f579bcd90bdc8 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 16:21:50 +0000 Subject: [PATCH 181/933] Revert "[RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657)" This reverts commit 2091c74796b1dac68e622284c63a870b88b7554f. Builtbot failure: https://lab.llvm.org/buildbot/#/builders/16/builds/63080 --- llvm/docs/RemoveDIsDebugInfo.md | 11 +-- llvm/include/llvm-c/DebugInfo.h | 60 +++++--------- llvm/lib/IR/DebugInfo.cpp | 121 ++++++++++------------------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 68 insertions(+), 137 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index 9e50a2a604aa..a2f1e173d9d9 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,22 +40,15 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. - -Existing functions (behaviour change) -------------------------------------- -LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). -LLVMDIBuilderInsertDeclareAtEnd # Same as above. -LLVMDIBuilderInsertDbgValueBefore # Same as above. -LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index dab1d697761b..b23ff63c862f 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,12 +1249,7 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new Declare DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1262,13 +1257,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1284,7 +1279,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1300,14 +1295,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Declare DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1315,12 +1305,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1338,7 +1328,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1356,12 +1346,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new Value DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1369,13 +1354,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1391,7 +1376,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1407,14 +1392,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Value DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1422,12 +1402,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1445,7 +1425,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 4206162d1768..09bce9df1f33 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, - DL, Instr); + return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, + Expr, DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,38 +1679,27 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap( + unwrap(Builder) + ->insertDeclare(unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, - DL, Block); + return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, + Expr, DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1718,36 +1707,26 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDeclare(unwrap(Storage), + unwrap(VarInfo), + unwrap(Expr), + unwrap(DL), unwrap(Block)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1755,36 +1734,26 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1792,29 +1761,19 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Block)) + .get()); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 9b5c37b05d90..78ccaf12a380 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,13 +136,12 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); - if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -150,11 +149,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -162,11 +161,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 8155ec13968b6457c61b8507f2ae8ba3ac3b748b Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Fri, 22 Mar 2024 12:30:02 -0400 Subject: [PATCH 182/933] [HIP][NFC] Refactor managed var codegen (#85976) Refactor managed variable handling in codegen so that the transformation is done separately from registration. This will allow the new driver to register the managed var in the linker wrapper. --- clang/lib/CodeGen/CGCUDANV.cpp | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index d3f2573fd5e3..b756318c46a9 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -605,20 +605,10 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(Var->getValueType()); if (Info.Flags.isManaged()) { - auto *ManagedVar = new llvm::GlobalVariable( - CGM.getModule(), Var->getType(), - /*isConstant=*/false, Var->getLinkage(), - /*Init=*/Var->isDeclaration() - ? nullptr - : llvm::ConstantPointerNull::get(Var->getType()), - /*Name=*/"", /*InsertBefore=*/nullptr, - llvm::GlobalVariable::NotThreadLocal); - ManagedVar->setDSOLocal(Var->isDSOLocal()); - ManagedVar->setVisibility(Var->getVisibility()); - ManagedVar->setExternallyInitialized(true); - ManagedVar->takeName(Var); - Var->setName(Twine(ManagedVar->getName() + ".managed")); - replaceManagedVar(Var, ManagedVar); + assert(Var->getName().ends_with(".managed") && + "HIP managed variables not transformed"); + auto *ManagedVar = CGM.getModule().getNamedGlobal( + Var->getName().drop_back(StringRef(".managed").size())); llvm::Value *Args[] = { &GpuBinaryHandlePtr, ManagedVar, @@ -1093,7 +1083,9 @@ void CGNVCUDARuntime::transformManagedVars() { : llvm::ConstantPointerNull::get(Var->getType()), /*Name=*/"", /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - CGM.getContext().getTargetAddressSpace(LangAS::cuda_device)); + CGM.getContext().getTargetAddressSpace(CGM.getLangOpts().CUDAIsDevice + ? LangAS::cuda_device + : LangAS::Default)); ManagedVar->setDSOLocal(Var->isDSOLocal()); ManagedVar->setVisibility(Var->getVisibility()); ManagedVar->setExternallyInitialized(true); @@ -1102,7 +1094,7 @@ void CGNVCUDARuntime::transformManagedVars() { Var->setName(Twine(ManagedVar->getName()) + ".managed"); // Keep managed variables even if they are not used in device code since // they need to be allocated by the runtime. - if (!Var->isDeclaration()) { + if (CGM.getLangOpts().CUDAIsDevice && !Var->isDeclaration()) { assert(!ManagedVar->isDeclaration()); CGM.addCompilerUsedGlobal(Var); CGM.addCompilerUsedGlobal(ManagedVar); @@ -1160,9 +1152,8 @@ void CGNVCUDARuntime::createOffloadingEntries() { // Returns module constructor to be added. llvm::Function *CGNVCUDARuntime::finalizeModule() { + transformManagedVars(); if (CGM.getLangOpts().CUDAIsDevice) { - transformManagedVars(); - // Mark ODR-used device variables as compiler used to prevent it from being // eliminated by optimization. This is necessary for device variables // ODR-used by host functions. Sema correctly marks them as ODR-used no -- GitLab From 7269570e4b2a5197201c959652c3e86804ed1eeb Mon Sep 17 00:00:00 2001 From: Abhin P Jose Date: Fri, 22 Mar 2024 17:39:43 +0100 Subject: [PATCH 183/933] Fixed build breaking due to #77178 and #86131 (#86290) Fixed a small issue of matching pthread signature, which was causing the build to break for the compiler-rt project after adding -Wcast-function-type-mismatch to -Wextra dignostic group (https://github.com/llvm/llvm-project/pull/77178 & https://github.com/llvm/llvm-project/pull/86131). --- compiler-rt/lib/asan/tests/asan_noinst_test.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp index 4c103609c83b..df7de2d7d15e 100644 --- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp +++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp @@ -45,7 +45,8 @@ TEST(AddressSanitizer, InternalSimpleDeathTest) { EXPECT_DEATH(exit(1), ""); } -static void MallocStress(size_t n) { +static void *MallocStress(void *NumOfItrPtr) { + size_t n = *((size_t *)NumOfItrPtr); u32 seed = my_rand(); BufferedStackTrace stack1; stack1.trace_buffer[0] = 0xa123; @@ -90,20 +91,21 @@ static void MallocStress(size_t n) { } for (size_t i = 0; i < vec.size(); i++) __asan::asan_free(vec[i], &stack3, __asan::FROM_MALLOC); + return nullptr; } - TEST(AddressSanitizer, NoInstMallocTest) { - MallocStress(ASAN_LOW_MEMORY ? 300000 : 1000000); + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 300000 : 1000000; + MallocStress((void *)&kNumIterations); } TEST(AddressSanitizer, ThreadedMallocStressTest) { const int kNumThreads = 4; - const int kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; pthread_t t[kNumThreads]; for (int i = 0; i < kNumThreads; i++) { - PTHREAD_CREATE(&t[i], 0, (void* (*)(void *x))MallocStress, - (void*)kNumIterations); + PTHREAD_CREATE(&t[i], 0, (void *(*)(void *x))MallocStress, + (void *)&kNumIterations); } for (int i = 0; i < kNumThreads; i++) { PTHREAD_JOIN(t[i], 0); -- GitLab From d2f684685afeffcffba7e889e7267bce1d905911 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:48:25 -0400 Subject: [PATCH 184/933] [C99] Update status of DR290, which we do not yet implement --- clang/test/C/drs/dr290.c | 20 ++++++++++++++++++++ clang/www/c_dr_status.html | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/drs/dr290.c diff --git a/clang/test/C/drs/dr290.c b/clang/test/C/drs/dr290.c new file mode 100644 index 000000000000..3a6fd1d0dab6 --- /dev/null +++ b/clang/test/C/drs/dr290.c @@ -0,0 +1,20 @@ +/* RUN: %clang_cc1 -fsyntax-only -ast-dump %s | FileCheck %s + */ + +/* WG14 DR290: no + * FLT_EVAL_METHOD and extra precision and/or range + * + * We retain an implicit conversion based on the float eval method being used + * instead of dropping it due to the explicit cast. See GH86304 and C23 6.5.5p7. + */ + +#pragma clang fp eval_method(double) +_Static_assert((float)(123.0F * 2.0F) == (float)246.0F, ""); + +// CHECK: StaticAssertDecl +// CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' +// CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' +// NB: the following implicit cast is incorrect. +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 +// CHECK-NEXT: CStyleCastExpr {{.*}} 'float' FPEvalMethod=1 + diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index ed45123ffd0e..a41c4f717067 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -1686,7 +1686,7 @@ conformance.

290 C99 FLT_EVAL_METHOD and extra precision and/or range - Unknown + No 291 -- GitLab From 527a624205748814dd9309eda7ee308b40b2359a Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:49:12 -0400 Subject: [PATCH 185/933] [C11] Update the status of N1365 on constant expression handling This paper is about constant expression handling in the presence of FLT_EVAL_METHOD, which we handle via insertion of implicit cast nodes in the AST. --- clang/test/C/C11/n1365.c | 60 ++++++++++++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C11/n1365.c diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c new file mode 100644 index 000000000000..3f769faa365d --- /dev/null +++ b/clang/test/C/C11/n1365.c @@ -0,0 +1,60 @@ +// RUN: %clang_cc1 -ast-dump %s | FileCheck %s + +/* WG14 N1365: Clang 16 + * Constant expressions + */ + +// Note: we don't allow you to expand __FLT_EVAL_METHOD__ in the presence of a +// pragma that changes its value. However, we can test that we have the correct +// constant expression behavior by testing that the AST has the correct implicit +// casts, which also specify that the cast was inserted due to an evaluation +// method requirement. +void func(void) { + { + #pragma clang fp eval_method(double) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + + // Ensure that a cast removes the extra precision. + _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(extended) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'long double' '*' FPEvalMethod=2 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(source) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'float' '*' FPEvalMethod=0 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: FloatingLiteral + } +} diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 0069da74cbd5..f00d5a6b7094 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -471,7 +471,7 @@ conformance.

Constant expressions N1365 - Unknown + Full Contractions and expression evaluation methods -- GitLab From 72c729f354d71697a1402720c90b57ff521b6739 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:51:20 -0700 Subject: [PATCH 186/933] [bazel] Add support for --incompatible_disallow_empty_glob (#85999) This bazel flag, that should be flipped in an upcoming release https://github.com/bazelbuild/bazel/pull/15327, fails if globs have no matches. This helps find libraries where you are accidentally not including files because of typos. This change removes the various globs that were not matching anything, and uncovered some targets that were doing nothing because their source files were deleted. There are a few cases where globs were intentionally optional in the case of loops that expanded to different potential options, so those now use `allow_empty = True`. This allows downstream consumers to also flip this flags for their own builds, where previously this would fail in LLVM instead. The downside to this change is that if files are added in these relatively standard locations, manual work will have to be done to add this patterns back. If folks prefer we could instead add `allow_empty = True` to every glob. --- utils/bazel/.bazelrc | 4 + .../llvm-project-overlay/bolt/BUILD.bazel | 2 - .../clang-tools-extra/clang-tidy/defs.bzl | 4 +- .../llvm-project-overlay/clang/BUILD.bazel | 22 +- .../llvm-project-overlay/lld/BUILD.bazel | 2 - .../llvm-project-overlay/llvm/BUILD.bazel | 337 +++++++----------- .../llvm-project-overlay/mlir/BUILD.bazel | 144 -------- .../mlir/test/BUILD.bazel | 13 - .../mlir/unittests/BUILD.bazel | 38 -- 9 files changed, 138 insertions(+), 428 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index 46894decc7ad..e8d055ec2322 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -35,6 +35,10 @@ build --features=layering_check # See: https://bazel.build/reference/be/functions#exports_files build --incompatible_no_implicit_file_export +# Enable so downstream users can flip this flag globally, this should +# eventually become the default +common --incompatible_disallow_empty_glob + ############################################################################### # Options to select different strategies for linking potential dependent # libraries. The default leaves it disabled. diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel index 043a3b61a75f..1c12c8167ba4 100644 --- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel @@ -221,8 +221,6 @@ cc_library( srcs = glob([ "lib/Target/AArch64/*.cpp", ]), - hdrs = glob([ - ]), includes = ["include"], deps = [ ":Core", diff --git a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl index 41c03aad871c..5abe4b08f5d9 100644 --- a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl +++ b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl @@ -16,8 +16,8 @@ _common_library_deps = [ ] def clang_tidy_library(name, **kwargs): - kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")])) - kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")])) + kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")], allow_empty = True)) + kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")], allow_empty=True)) kwargs["deps"] = kwargs.get("deps", []) + _common_library_deps cc_library( name = name, diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 865cafbf50c6..c01986815afe 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -614,7 +614,6 @@ cc_library( "include/clang/Basic/Version.inc", ] + glob([ "lib/Basic/*.cpp", - "lib/Basic/*.c", "lib/Basic/*.h", "lib/Basic/Targets/*.cpp", "lib/Basic/Targets/*.h", @@ -1042,7 +1041,6 @@ cc_library( "lib/Analysis/FlowSensitive/Models/*.cpp", "lib/Analysis/FlowSensitive/*.cpp", "lib/Analysis/*.cpp", - "lib/Analysis/*.h", ]) + [ ":analysis_htmllogger_gen", ], @@ -1180,10 +1178,8 @@ gentbl( cc_library( name = "parse", - srcs = [ - ] + glob([ + srcs = glob([ "lib/Parse/*.cpp", - "lib/Parse/*.h", ]), hdrs = [ "include/clang/Parse/AttrParserStringSwitches.inc", @@ -1207,7 +1203,6 @@ cc_library( name = "ast_matchers", srcs = glob([ "lib/ASTMatchers/*.cpp", - "lib/ASTMatchers/*.h", ]), hdrs = glob(["include/clang/ASTMatchers/*.h"]), includes = ["include"], @@ -1241,7 +1236,6 @@ cc_library( name = "rewrite", srcs = glob([ "lib/Rewrite/*.cpp", - "lib/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Core/*.h"]), includes = ["include"], @@ -1275,7 +1269,6 @@ cc_library( name = "tooling_core", srcs = glob([ "lib/Tooling/Core/*.cpp", - "lib/Tooling/Core/*.h", ]), hdrs = glob(["include/clang/Tooling/Core/*.h"]), includes = ["include"], @@ -1340,11 +1333,9 @@ cc_library( name = "tooling_refactoring", srcs = glob([ "lib/Tooling/Refactoring/**/*.cpp", - "lib/Tooling/Refactoring/**/*.h", ]), hdrs = glob([ "include/clang/Tooling/Refactoring/**/*.h", - "include/clang/Tooling/Refactoring/**/*.def", ]), deps = [ ":ast", @@ -1593,9 +1584,6 @@ cc_library( srcs = glob( [ "lib/Driver/*.cpp", - "lib/Driver/*.h", - "lib/Driver/Arch/*.cpp", - "lib/Driver/Arch/*.h", "lib/Driver/ToolChains/*.cpp", "lib/Driver/ToolChains/*.h", "lib/Driver/ToolChains/Arch/*.cpp", @@ -1833,9 +1821,6 @@ cc_library( copts = ["$(STACK_FRAME_UNLIMITED)"], data = [":builtin_headers_gen"], includes = ["include"], - textual_hdrs = glob([ - "include/clang/Frontend/*.def", - ]), deps = [ ":apinotes", ":ast", @@ -1872,7 +1857,6 @@ cc_library( name = "frontend_rewrite", srcs = glob([ "lib/Frontend/Rewrite/*.cpp", - "lib/Frontend/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Frontend/*.h"]), includes = ["include"], @@ -2116,7 +2100,6 @@ cc_library( name = "frontend_tool", srcs = glob([ "lib/FrontendTool/*.cpp", - "lib/FrontendTool/*.h", ]), hdrs = glob(["include/clang/FrontendTool/*.h"]), includes = ["include"], @@ -2320,7 +2303,6 @@ cc_binary( testonly = 1, srcs = glob([ "tools/clang-import-test/*.cpp", - "tools/clang-import-test/*.h", ]), stamp = 0, deps = [ @@ -2350,7 +2332,6 @@ cc_library( name = "clang-driver", srcs = glob([ "tools/driver/*.cpp", - "tools/driver/*.h", ]) + ["clang-driver.cpp"], copts = [ # Disable stack frame size checks in the driver because @@ -2668,7 +2649,6 @@ cc_library( name = "extract_api", srcs = glob([ "lib/ExtractAPI/**/*.cpp", - "lib/ExtractAPI/**/*.h", ]), hdrs = glob(["include/clang/ExtractAPI/**/*.h"]), includes = ["include"], diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel index 8fb71fc1f971..5a494a13acea 100644 --- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel @@ -187,7 +187,6 @@ cc_library( name = "MinGW", srcs = glob([ "MinGW/*.cpp", - "MinGW/*.h", ]), includes = ["MinGW"], deps = [ @@ -296,7 +295,6 @@ cc_binary( name = "lld", srcs = glob([ "tools/lld/*.cpp", - "tools/lld/*.h", ]) + ["lld-driver.cpp"], deps = [ ":COFF", diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 07c5a00c07d7..f5a2d264b690 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -66,7 +66,10 @@ enum_targets_gen( llvm_target_asm_parsers = [ t for t in llvm_targets - if glob(["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -81,7 +84,10 @@ enum_targets_gen( llvm_target_disassemblers = [ t for t in llvm_targets - if glob(["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -96,7 +102,10 @@ enum_targets_gen( llvm_target_mcas = [ t for t in llvm_targets - if glob(["lib/Target/{}/MCA/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/MCA/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -111,7 +120,10 @@ enum_targets_gen( llvm_target_exegesis = [ t for t in llvm_targets - if glob(["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)]) + if glob( + ["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -168,7 +180,6 @@ cc_library( name = "Demangle", srcs = glob([ "lib/Demangle/*.cpp", - "lib/Demangle/*.h", ]), hdrs = glob([ "include/llvm/Demangle/*.h", @@ -203,7 +214,6 @@ cc_library( "include/llvm/Option/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/Support/Windows/*.h", "lib/Support/Windows/*.inc", ]), "//conditions:default": glob([ @@ -315,7 +325,6 @@ cc_library( name = "LineEditor", srcs = glob([ "lib/LineEditor/*.cpp", - "lib/LineEditor/*.h", ]), hdrs = glob(["include/llvm/LineEditor/*.h"]), copts = llvm_copts, @@ -329,7 +338,6 @@ cc_library( name = "Option", srcs = glob([ "lib/Option/*.cpp", - "lib/Option/*.h", ]), hdrs = glob(["include/llvm/Option/*.h"]), copts = llvm_copts, @@ -376,8 +384,6 @@ cc_library( name = "BinaryFormat", srcs = glob([ "lib/BinaryFormat/*.cpp", - "lib/BinaryFormat/*.def", - "lib/BinaryFormat/*.h", ]), hdrs = glob([ "include/llvm/BinaryFormat/*.h", @@ -409,7 +415,6 @@ cc_library( name = "DebugInfoMSF", srcs = glob([ "lib/DebugInfo/MSF/*.cpp", - "lib/DebugInfo/MSF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/MSF/*.h"]), copts = llvm_copts, @@ -420,7 +425,6 @@ cc_library( name = "DebugInfoBTF", srcs = glob([ "lib/DebugInfo/BTF/*.cpp", - "lib/DebugInfo/BTF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/BTF/*.h"]) + [ "include/llvm/DebugInfo/BTF/BTF.def", @@ -437,7 +441,6 @@ cc_library( name = "DebugInfoCodeView", srcs = glob([ "lib/DebugInfo/CodeView/*.cpp", - "lib/DebugInfo/CodeView/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/CodeView/*.h", @@ -480,9 +483,7 @@ cc_library( name = "DebugInfoPDB", srcs = glob([ "lib/DebugInfo/PDB/*.cpp", - "lib/DebugInfo/PDB/*.h", "lib/DebugInfo/PDB/Native/*.cpp", - "lib/DebugInfo/PDB/Native/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/PDB/*.h", @@ -523,12 +524,9 @@ cc_library( name = "MC", srcs = glob([ "lib/MC/*.cpp", - "lib/MC/*.h", ]), hdrs = glob([ "include/llvm/MC/*.h", - "include/llvm/MC/*.def", - "include/llvm/MC/*.inc", ]), copts = llvm_copts, deps = [ @@ -545,7 +543,6 @@ cc_library( name = "DebugInfoDWARF", srcs = glob([ "lib/DebugInfo/DWARF/*.cpp", - "lib/DebugInfo/DWARF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/DWARF/*.h"]), copts = llvm_copts, @@ -563,7 +560,6 @@ cc_library( name = "DebugInfoGSYM", srcs = glob([ "lib/DebugInfo/GSYM/*.cpp", - "lib/DebugInfo/GSYM/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/GSYM/*.h"]), copts = llvm_copts, @@ -580,7 +576,6 @@ cc_library( name = "Symbolize", srcs = glob([ "lib/DebugInfo/Symbolize/*.cpp", - "lib/DebugInfo/Symbolize/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/Symbolize/*.h", @@ -658,7 +653,6 @@ cc_binary( srcs = glob( [ "utils/TableGen/*.cpp", - "utils/TableGen/*.inc", "utils/TableGen/*.h", "utils/TableGen/GlobalISel/*.cpp", "utils/TableGen/GlobalISel/*.h", @@ -821,7 +815,6 @@ cc_library( name = "BitstreamReader", srcs = glob([ "lib/Bitstream/Reader/*.cpp", - "lib/Bitstream/Reader/*.h", ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", @@ -836,9 +829,6 @@ cc_library( cc_library( name = "BitstreamWriter", - srcs = glob([ - "lib/Bitstream/Writer/*.h", - ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", "include/llvm/Bitstream/BitCodes.h", @@ -956,7 +946,6 @@ cc_library( name = "MCParser", srcs = glob([ "lib/MC/MCParser/*.cpp", - "lib/MC/MCParser/*.h", ]), hdrs = glob(["include/llvm/MC/MCParser/*.h"]), copts = llvm_copts, @@ -1002,9 +991,7 @@ cc_library( srcs = glob([ "lib/TextAPI/BinaryReader/**/*.cpp", ]), - hdrs = ["include/llvm/TextAPI/DylibReader.h"] + glob( - ["lib/TextAPI/BinaryReader/**/*.h"], - ), + hdrs = ["include/llvm/TextAPI/DylibReader.h"], copts = llvm_copts, deps = [ ":Object", @@ -1067,7 +1054,6 @@ cc_library( name = "ObjectYAML", srcs = glob([ "lib/ObjectYAML/*.cpp", - "lib/ObjectYAML/*.h", ]), hdrs = glob(["include/llvm/ObjectYAML/*.h"]), copts = llvm_copts, @@ -1085,7 +1071,6 @@ cc_library( name = "ProfileData", srcs = glob([ "lib/ProfileData/*.cpp", - "lib/ProfileData/*.h", ]), hdrs = glob([ "include/llvm/ProfileData/*.h", @@ -1109,7 +1094,6 @@ cc_library( name = "Coverage", srcs = glob([ "lib/ProfileData/Coverage/*.cpp", - "lib/ProfileData/Coverage/*.h", ]), hdrs = glob(["include/llvm/ProfileData/Coverage/*.h"]), copts = llvm_copts, @@ -1126,8 +1110,6 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", - "lib/Analysis/*.def", ], ), hdrs = glob( @@ -1185,7 +1167,6 @@ cc_library( name = "Target", srcs = glob([ "lib/Target/*.cpp", - "lib/Target/*.h", ]), hdrs = glob([ "include/llvm/Target/*.h", @@ -1221,14 +1202,11 @@ cc_library( name = "TargetParser", srcs = glob([ "lib/TargetParser/*.cpp", - "lib/TargetParser/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/TargetParser/Windows/*.h", "lib/TargetParser/Windows/*.inc", ]), "//conditions:default": glob([ - "lib/TargetParser/Unix/*.h", "lib/TargetParser/Unix/*.inc", ]), }), @@ -1252,7 +1230,6 @@ cc_library( name = "DWP", srcs = glob([ "lib/DWP/*.cpp", - "lib/DWP/*.h", ]), hdrs = glob(["include/llvm/DWP/*.h"]), copts = llvm_copts, @@ -1269,7 +1246,6 @@ cc_library( name = "TransformUtils", srcs = glob([ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ]), hdrs = glob(["include/llvm/Transforms/Utils/*.h"]) + [ "include/llvm/Transforms/Utils.h", @@ -1390,7 +1366,6 @@ cc_library( name = "Scalar", srcs = glob([ "lib/Transforms/Scalar/*.cpp", - "lib/Transforms/Scalar/*.h", ]), hdrs = glob(["include/llvm/Transforms/Scalar/*.h"]) + [ "include/llvm/Transforms/Scalar.h", @@ -1432,9 +1407,6 @@ cc_library( cc_library( name = "FrontendDebug", - srcs = glob([ - "lib/Frontend/Debug/*.cpp", - ]), hdrs = glob([ "include/llvm/Frontend/Debug/*.h", ]), @@ -1530,8 +1502,6 @@ cc_library( ]), hdrs = glob([ "include/llvm/Frontend/OpenMP/*.h", - "include/llvm/Frontend/OpenMP/OMP/*.h", - "include/llvm/Frontend/*.h", ]) + [ "include/llvm/Frontend/OpenMP/OMP.h.inc", "include/llvm/Frontend/OpenMP/OMP.inc", @@ -1591,9 +1561,7 @@ cc_library( ]) + [ "include/llvm/Frontend/OpenACC/ACC.inc", ], - hdrs = glob([ - "include/llvm/Frontend/OpenACC/*.h", - ]) + ["include/llvm/Frontend/OpenACC/ACC.h.inc"], + hdrs = ["include/llvm/Frontend/OpenACC/ACC.h.inc"], copts = llvm_copts, deps = [ ":Analysis", @@ -1607,7 +1575,6 @@ cc_library( name = "AsmParser", srcs = glob([ "lib/AsmParser/*.cpp", - "lib/AsmParser/*.h", ]), hdrs = glob(["include/llvm/AsmParser/*.h"]), copts = llvm_copts, @@ -1623,7 +1590,6 @@ cc_library( name = "IRPrinter", srcs = glob([ "lib/IRPrinter/*.cpp", - "lib/IRPrinter/*.h", ]), hdrs = glob([ "include/llvm/IRPrinter/*.h", @@ -1640,7 +1606,6 @@ cc_library( name = "IRReader", srcs = glob([ "lib/IRReader/*.cpp", - "lib/IRReader/*.h", ]), hdrs = glob([ "include/llvm/IRReader/*.h", @@ -1683,7 +1648,6 @@ cc_library( name = "IPO", srcs = glob([ "lib/Transforms/IPO/*.cpp", - "lib/Transforms/IPO/*.h", ]), hdrs = glob([ "include/llvm/Transforms/IPO/*.h", @@ -1721,7 +1685,6 @@ cc_library( name = "CFGuard", srcs = glob([ "lib/Transforms/CFGuard/*.cpp", - "lib/Transforms/CFGuard/*.h", ]), hdrs = ["include/llvm/Transforms/CFGuard.h"], copts = llvm_copts, @@ -1736,7 +1699,6 @@ cc_library( name = "HipStdPar", srcs = glob([ "lib/Transforms/HipStdPar/*.cpp", - "lib/Transforms/HipStdPar/*.h", ]), hdrs = ["include/llvm/Transforms/HipStdPar/HipStdPar.h"], copts = llvm_copts, @@ -1826,7 +1788,6 @@ cc_library( copts = llvm_copts, textual_hdrs = glob([ "include/llvm/CodeGen/**/*.def", - "include/llvm/CodeGen/**/*.inc", ]), deps = [ ":AggressiveInstCombine", @@ -2305,10 +2266,13 @@ gentbl( td_file = "lib/Target/" + target["name"] + "/" + target["short_name"] + ".td", td_srcs = [ ":common_target_td_sources", - ] + glob([ - "lib/Target/" + target["name"] + "/*.td", - "lib/Target/" + target["name"] + "/GISel/*.td", - ]), + ] + glob( + [ + "lib/Target/" + target["name"] + "/*.td", + "lib/Target/" + target["name"] + "/GISel/*.td", + ], + allow_empty = True, + ), deps = target.get("tbl_deps", []), )], [cc_library( @@ -2332,43 +2296,49 @@ gentbl( # a number of targets due to crisscrossing inclusion of headers. [cc_library( name = target["name"] + "UtilsAndDesc", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", - "lib/Target/" + target["name"] + "/Utils/*.cpp", - - # We have to include these headers here as well as in the `hdrs` - # below to allow the `.cpp` files to use file-relative-inclusion to - # find them, even though consumers of this library use inclusion - # relative to the target with the `strip_includes_prefix` of this - # library. This mixture is likely incompatible with header modules. - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - ]), - hdrs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - - # This a bit of a hack to allow us to expose common, internal - # target header files to other libraries within the target via - # target-relative includes. This usage of headers is inherently - # non-modular as there is a mixture of target-relative inclusion - # using this rule and file-relative inclusion using the repeated - # listing of these headers in the `srcs` of subsequent rules. - "lib/Target/" + target["name"] + "/*.h", - - # FIXME: The entries below should be `textual_hdrs` instead of - # `hdrs`, but unfortunately that doesn't work with - # `strip_include_prefix`: - # https://github.com/bazelbuild/bazel/issues/12424 - # - # Once that issue is fixed and released, we can switch this to - # `textual_hdrs` and remove the feature disabling the various Bazel - # features (both current and under-development) that motivated the - # distinction between these two. - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", + "lib/Target/" + target["name"] + "/Utils/*.cpp", + + # We have to include these headers here as well as in the `hdrs` + # below to allow the `.cpp` files to use file-relative-inclusion to + # find them, even though consumers of this library use inclusion + # relative to the target with the `strip_includes_prefix` of this + # library. This mixture is likely incompatible with header modules. + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + ], + allow_empty = True, + ), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + + # This a bit of a hack to allow us to expose common, internal + # target header files to other libraries within the target via + # target-relative includes. This usage of headers is inherently + # non-modular as there is a mixture of target-relative inclusion + # using this rule and file-relative inclusion using the repeated + # listing of these headers in the `srcs` of subsequent rules. + "lib/Target/" + target["name"] + "/*.h", + + # FIXME: The entries below should be `textual_hdrs` instead of + # `hdrs`, but unfortunately that doesn't work with + # `strip_include_prefix`: + # https://github.com/bazelbuild/bazel/issues/12424 + # + # Once that issue is fixed and released, we can switch this to + # `textual_hdrs` and remove the feature disabling the various Bazel + # features (both current and under-development) that motivated the + # distinction between these two. + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", + ], + allow_empty = True, + ), copts = llvm_copts, features = [ "-parse_headers", @@ -2392,20 +2362,26 @@ gentbl( )], [cc_library( name = target["name"] + "CodeGen", - srcs = glob([ - "lib/Target/" + target["name"] + "/GISel/*.cpp", - "lib/Target/" + target["name"] + "/GISel/*.h", - "lib/Target/" + target["name"] + "/*.cpp", - "lib/Target/" + target["name"] + "/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/GISel/*.cpp", + "lib/Target/" + target["name"] + "/GISel/*.h", + "lib/Target/" + target["name"] + "/*.cpp", + "lib/Target/" + target["name"] + "/*.h", + ], + allow_empty = True, + ), hdrs = ["lib/Target/" + target["name"] + "/" + target["short_name"] + ".h"], copts = llvm_copts, features = ["-layering_check"], strip_include_prefix = "lib/Target/" + target["name"], - textual_hdrs = glob([ - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - ]), + textual_hdrs = glob( + [ + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + ], + allow_empty = True, + ), deps = [ ":Analysis", ":BinaryFormat", @@ -2430,10 +2406,13 @@ gentbl( )], [cc_library( name = target["name"] + "AsmParser", - srcs = glob([ - "lib/Target/" + target["name"] + "/AsmParser/*.cpp", - "lib/Target/" + target["name"] + "/AsmParser/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/AsmParser/*.cpp", + "lib/Target/" + target["name"] + "/AsmParser/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, deps = [ ":BinaryFormat", @@ -2464,9 +2443,12 @@ gentbl( # `textual_hdrs` and remove the feature disabling the various Bazel # features (both current and under-development) that motivated the # distinction between these two. - hdrs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), features = [ "-parse_headers", "-header_modules", @@ -2475,11 +2457,14 @@ gentbl( )], [cc_library( name = target["name"] + "Disassembler", - srcs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.cpp", - "lib/Target/" + target["name"] + "/Disassembler/*.c", - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.cpp", + "lib/Target/" + target["name"] + "/Disassembler/*.c", + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2497,11 +2482,14 @@ gentbl( )], [cc_library( name = target["name"] + "TargetMCA", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCA/*.cpp", - "lib/Target/" + target["name"] + "/MCA/*.c", - "lib/Target/" + target["name"] + "/MCA/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCA/*.cpp", + "lib/Target/" + target["name"] + "/MCA/*.c", + "lib/Target/" + target["name"] + "/MCA/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2559,28 +2547,10 @@ cc_library( textual_hdrs = ["lib/Passes/PassRegistry.def"], ) -cc_library( - name = "MLPolicies", - srcs = glob([ - "lib/Analysis/ML/*.cpp", - "lib/Analysis/ML/*.h", - ]), - hdrs = glob([ - "include/llvm/Analysis/ML/*.h", - ]), - copts = llvm_copts, - deps = [ - ":Analysis", - ":Core", - ":Support", - ], -) - cc_library( name = "Passes", srcs = glob([ "lib/Passes/*.cpp", - "lib/Passes/*.h", ]), hdrs = glob([ "include/llvm/Passes/*.h", @@ -2601,7 +2571,6 @@ cc_library( ":InstCombine", ":Instrumentation", ":MC", - ":MLPolicies", ":ObjCARC", ":Scalar", ":Support", @@ -2618,7 +2587,6 @@ cc_library( name = "LTO", srcs = glob([ "lib/LTO/*.cpp", - "lib/LTO/*.h", ]), hdrs = glob([ "include/llvm/LTO/*.h", @@ -2658,7 +2626,6 @@ cc_library( name = "ExecutionEngine", srcs = glob([ "lib/ExecutionEngine/*.cpp", - "lib/ExecutionEngine/*.h", "lib/ExecutionEngine/RuntimeDyld/*.cpp", "lib/ExecutionEngine/RuntimeDyld/*.h", "lib/ExecutionEngine/RuntimeDyld/Targets/*.cpp", @@ -2772,11 +2739,9 @@ cc_library( name = "OrcJIT", srcs = glob([ "lib/ExecutionEngine/Orc/*.cpp", - "lib/ExecutionEngine/Orc/*.h", ]), hdrs = glob([ "include/llvm/ExecutionEngine/Orc/*.h", - "include/llvm/ExecutionEngine/Orc/RPC/*.h", ]) + [ "include/llvm-c/LLJIT.h", "include/llvm-c/Orc.h", @@ -2900,7 +2865,6 @@ cc_library( name = "DWARFLinker", srcs = glob([ "lib/DWARFLinker/Classic/*.cpp", - "lib/DWARFLinker/Classic/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/Classic/*.h"]), copts = llvm_copts, @@ -2921,7 +2885,6 @@ cc_library( name = "DWARFLinkerBase", srcs = glob([ "lib/DWARFLinker/*.cpp", - "lib/DWARFLinker/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/*.h"]), copts = llvm_copts, @@ -3012,7 +2975,6 @@ cc_library( name = "InterfaceStub", srcs = glob([ "lib/InterfaceStub/*.cpp", - "lib/InterfaceStub/*.h", ]), hdrs = glob([ "include/llvm/InterfaceStub/*.h", @@ -3063,7 +3025,6 @@ cc_library( name = "MCA", srcs = glob([ "lib/MCA/**/*.cpp", - "lib/MCA/**/*.h", ]), hdrs = glob([ "include/llvm/MCA/**/*.h", @@ -3090,7 +3051,6 @@ cc_library( name = "XRay", srcs = glob([ "lib/XRay/*.cpp", - "lib/XRay/*.h", ]), hdrs = glob(["include/llvm/XRay/*.h"]), copts = llvm_copts, @@ -3147,21 +3107,24 @@ cc_library( cc_library( name = "Exegesis", - srcs = glob([ - "tools/llvm-exegesis/lib/*.cpp", - # We have to include these headers here as well as in the `hdrs` below - # to allow the `.cpp` files to use file-relative-inclusion to find - # them, even though consumers of this library use inclusion relative to - # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this - # library. This mixture appears to be incompatible with header modules. - "tools/llvm-exegesis/lib/*.h", - ] + [ - "tools/llvm-exegesis/lib/{}/*.cpp".format(t) - for t in llvm_target_exegesis - ] + [ - "tools/llvm-exegesis/lib/{}/*.h".format(t) - for t in llvm_target_exegesis - ]), + srcs = glob( + [ + "tools/llvm-exegesis/lib/*.cpp", + # We have to include these headers here as well as in the `hdrs` below + # to allow the `.cpp` files to use file-relative-inclusion to find + # them, even though consumers of this library use inclusion relative to + # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this + # library. This mixture appears to be incompatible with header modules. + "tools/llvm-exegesis/lib/*.h", + ] + [ + "tools/llvm-exegesis/lib/{}/*.cpp".format(t) + for t in llvm_target_exegesis + ] + [ + "tools/llvm-exegesis/lib/{}/*.h".format(t) + for t in llvm_target_exegesis + ], + allow_empty = True, + ), hdrs = glob(["tools/llvm-exegesis/lib/*.h"]), copts = llvm_copts, features = [ @@ -3335,7 +3298,6 @@ cc_binary( name = "llvm-ar", srcs = glob([ "tools/llvm-ar/*.cpp", - "tools/llvm-ar/*.h", ]) + ["llvm-ar-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3373,7 +3335,6 @@ cc_binary( name = "llvm-as", srcs = glob([ "tools/llvm-as/*.cpp", - "tools/llvm-as/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3390,7 +3351,6 @@ cc_binary( name = "llvm-bcanalyzer", srcs = glob([ "tools/llvm-bcanalyzer/*.cpp", - "tools/llvm-bcanalyzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3477,7 +3437,6 @@ cc_binary( name = "llvm-cvtres", srcs = glob([ "tools/llvm-cvtres/*.cpp", - "tools/llvm-cvtres/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3511,7 +3470,6 @@ cc_binary( name = "llvm-cxxmap", srcs = glob([ "tools/llvm-cxxmap/*.cpp", - "tools/llvm-cxxmap/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3546,7 +3504,6 @@ cc_binary( name = "llvm-cxxfilt", srcs = glob([ "tools/llvm-cxxfilt/*.cpp", - "tools/llvm-cxxfilt/*.h", ]) + ["llvm-cxxfilt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3579,7 +3536,6 @@ cc_binary( name = "llvm-debuginfod-find", srcs = glob([ "tools/llvm-debuginfod-find/*.cpp", - "tools/llvm-debuginfod-find/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3596,7 +3552,6 @@ cc_binary( name = "llvm-dis", srcs = glob([ "tools/llvm-dis/*.cpp", - "tools/llvm-dis/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3691,7 +3646,6 @@ cc_binary( name = "llvm-dwp", srcs = glob([ "tools/llvm-dwp/*.cpp", - "tools/llvm-dwp/*.h", ]) + ["llvm-dwp-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3731,7 +3685,6 @@ cc_binary( name = "llvm-extract", srcs = glob([ "tools/llvm-extract/*.cpp", - "tools/llvm-extract/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3773,7 +3726,6 @@ cc_binary( name = "llvm-gsymutil", srcs = glob([ "tools/llvm-gsymutil/*.cpp", - "tools/llvm-gsymutil/*.h", ]) + ["llvm-gsymutil-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3925,7 +3877,6 @@ cc_binary( name = "llvm-link", srcs = glob([ "tools/llvm-link/*.cpp", - "tools/llvm-link/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3990,7 +3941,6 @@ cc_binary( name = "llvm-lto", srcs = glob([ "tools/llvm-lto/*.cpp", - "tools/llvm-lto/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4013,7 +3963,6 @@ cc_binary( name = "llvm-lto2", srcs = glob([ "tools/llvm-lto2/*.cpp", - "tools/llvm-lto2/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4159,7 +4108,6 @@ cc_binary( name = "llvm-mt", srcs = glob([ "tools/llvm-mt/*.cpp", - "tools/llvm-mt/*.h", ]) + ["llvm-mt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4196,7 +4144,6 @@ cc_binary( name = "llvm-nm", srcs = glob([ "tools/llvm-nm/*.cpp", - "tools/llvm-nm/*.h", ]) + ["llvm-nm-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4280,7 +4227,6 @@ cc_binary( name = "llvm-stress", srcs = glob([ "tools/llvm-stress/*.cpp", - "tools/llvm-stress/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4457,7 +4403,6 @@ cc_binary( name = "llvm-profdata", srcs = glob([ "tools/llvm-profdata/*.cpp", - "tools/llvm-profdata/*.h", ]) + ["llvm-profdata-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4641,7 +4586,6 @@ cc_binary( name = "llvm-rtdyld", srcs = glob([ "tools/llvm-rtdyld/*.cpp", - "tools/llvm-rtdyld/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4683,7 +4627,6 @@ cc_binary( name = "llvm-size", srcs = glob([ "tools/llvm-size/*.cpp", - "tools/llvm-size/*.h", ]) + ["llvm-size-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4699,7 +4642,6 @@ cc_binary( name = "llvm-split", srcs = glob([ "tools/llvm-split/*.cpp", - "tools/llvm-split/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4729,7 +4671,6 @@ cc_binary( name = "llvm-strings", srcs = glob([ "tools/llvm-strings/*.cpp", - "tools/llvm-strings/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4766,7 +4707,6 @@ cc_binary( name = "llvm-symbolizer", srcs = glob([ "tools/llvm-symbolizer/*.cpp", - "tools/llvm-symbolizer/*.h", ]) + ["llvm-symbolizer-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4792,7 +4732,6 @@ cc_binary( name = "llvm-undname", srcs = glob([ "tools/llvm-undname/*.cpp", - "tools/llvm-undname/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4806,7 +4745,6 @@ cc_binary( name = "llvm-xray", srcs = glob([ "tools/llvm-xray/*.cpp", - "tools/llvm-xray/*.cc", "tools/llvm-xray/*.h", ]), copts = llvm_copts, @@ -4889,7 +4827,6 @@ cc_binary( name = "sancov", srcs = glob([ "tools/sancov/*.cpp", - "tools/sancov/*.h", ]) + ["sancov-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4912,7 +4849,6 @@ cc_binary( name = "sanstats", srcs = glob([ "tools/sanstats/*.cpp", - "tools/sanstats/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4927,7 +4863,6 @@ cc_binary( name = "split-file", srcs = glob([ "utils/split-file/*.cpp", - "utils/split-file/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5023,7 +4958,6 @@ cc_library( testonly = True, srcs = glob([ "lib/Testing/Support/*.cpp", - "lib/Testing/Support/*.h", ]), hdrs = glob(["include/llvm/Testing/Support/*.h"]), copts = llvm_copts, @@ -5052,7 +4986,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/FileCheck/*.cpp", - "utils/FileCheck/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5098,7 +5031,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/count/*.c", - "utils/count/*.h", ]), stamp = 0, deps = [":Support"], @@ -5109,7 +5041,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/lli/ChildTarget/*.cpp", - "tools/lli/ChildTarget/*.h", ]), copts = llvm_copts, # The tests load code into this binary that expect to see symbols @@ -5176,7 +5107,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-diff/*.cpp", - "tools/llvm-diff/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5194,7 +5124,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-isel-fuzzer/*.cpp", - "tools/llvm-isel-fuzzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5240,7 +5169,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/not/*.cpp", - "utils/not/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5320,7 +5248,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-tli-checker/*.cpp", - "tools/llvm-tli-checker/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5367,7 +5294,6 @@ cc_binary( name = "verify-uselistorder", srcs = glob([ "tools/verify-uselistorder/*.cpp", - "tools/verify-uselistorder/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5387,7 +5313,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/yaml2obj/*.cpp", - "tools/yaml2obj/*.h", ]), copts = llvm_copts, stamp = 0, diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 3b575d4a413c..5b6e4678a05e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -325,9 +325,7 @@ cc_library( "lib/IR/*.cpp", "lib/IR/*.h", "lib/IR/PDL/*.cpp", - "lib/Bytecode/Reader/*.h", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]) + [ "include/mlir/IR/PDLPatternMatch.h.inc", "include/mlir/Interfaces/CallInterfaces.h", @@ -1630,7 +1628,6 @@ cc_library( srcs = glob( [ "lib/Dialect/AMDGPU/Transforms/*.cpp", - "lib/Dialect/AMDGPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]), @@ -1769,7 +1766,6 @@ cc_library( name = "TargetCpp", srcs = glob([ "lib/Target/Cpp/*.cpp", - "lib/Target/Cpp/*.h", ]), hdrs = glob(["include/mlir/Target/Cpp/*.h"]), deps = [ @@ -1968,7 +1964,6 @@ cc_library( name = "ArmNeon2dToIntr", srcs = glob([ "lib/Conversion/ArmNeon2dToIntr/*.cpp", - "lib/Conversion/ArmNeon2dToIntr/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArmNeon2dToIntr/*.h", @@ -2840,7 +2835,6 @@ cc_library( name = "SCFTransforms", srcs = glob([ "lib/Dialect/SCF/Transforms/*.cpp", - "lib/Dialect/SCF/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/SCF/Transforms/*.h", @@ -3189,7 +3183,6 @@ cc_library( name = "SparseTensorTransforms", srcs = glob([ "lib/Dialect/SparseTensor/Transforms/*.cpp", - "lib/Dialect/SparseTensor/Transforms/*.h", "lib/Dialect/SparseTensor/Transforms/Utils/*.cpp", "lib/Dialect/SparseTensor/Transforms/Utils/*.h", ]), @@ -3829,7 +3822,6 @@ cc_library( name = "Dialect", srcs = glob([ "lib/Dialect/*.cpp", - "lib/Dialect/*.h", ]), hdrs = glob([ "include/mlir/Dialect/*.h", @@ -3871,7 +3863,6 @@ cc_library( name = "DialectUtils", srcs = glob([ "lib/Dialect/Utils/*.cpp", - "lib/Dialect/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Utils/*.h", @@ -3890,7 +3881,6 @@ cc_library( name = "AffineDialect", srcs = glob([ "lib/Dialect/Affine/IR/*.cpp", - "lib/Dialect/Affine/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Affine/IR/*.h", @@ -4012,7 +4002,6 @@ cc_library( name = "AffineAnalysis", srcs = glob([ "lib/Dialect/Affine/Analysis/*.cpp", - "lib/Dialect/Affine/Analysis/*.h", ]), hdrs = glob(["include/mlir/Dialect/Affine/Analysis/*.h"]), includes = ["include"], @@ -4036,7 +4025,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Affine/Utils/*.cpp", - "lib/Dialect/Affine/Utils/*.h", ], ), hdrs = [ @@ -4083,7 +4071,6 @@ cc_library( name = "AffineTransforms", srcs = glob([ "lib/Dialect/Affine/Transforms/*.cpp", - "lib/Dialect/Affine/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Affine/Passes.h", @@ -4191,7 +4178,6 @@ cc_library( ":MemRefToSPIRV", ":NVGPUToNVVM", ":NVVMToLLVM", - ":OpenACCToLLVM", ":OpenACCToSCF", ":OpenMPToLLVM", ":PDLToPDLInterp", @@ -4224,7 +4210,6 @@ cc_library( name = "AsyncToLLVM", srcs = glob([ "lib/Conversion/AsyncToLLVM/*.cpp", - "lib/Conversion/AsyncToLLVM/*.h", ]), hdrs = glob(["include/mlir/Conversion/AsyncToLLVM/*.h"]), includes = ["include"], @@ -4250,7 +4235,6 @@ cc_library( name = "AffineToStandard", srcs = glob([ "lib/Conversion/AffineToStandard/*.cpp", - "lib/Conversion/AffineToStandard/*.h", ]), hdrs = glob(["include/mlir/Conversion/AffineToStandard/*.h"]), includes = ["include"], @@ -4271,32 +4255,11 @@ cc_library( ], ) -# SDBM dialect only contains attribute components that can be constructed given -# a dialect object, so whenever it is used it must also be registered. Therefore -# we don't split out the registration library for it. -cc_library( - name = "SDBM", - srcs = glob([ - "lib/Dialect/SDBM/*.cpp", - "lib/Dialect/SDBM/*.h", - ]), - hdrs = glob([ - "include/mlir/Dialect/SDBM/*.h", - ]), - includes = ["include"], - deps = [ - ":IR", - ":Support", - "//llvm:Support", - ], -) - cc_library( name = "SCFDialect", srcs = glob( [ "lib/Dialect/SCF/IR/*.cpp", - "lib/Dialect/SCF/IR/*.h", ], ), hdrs = glob( @@ -4605,7 +4568,6 @@ cc_library( name = "ShapeToStandard", srcs = glob([ "lib/Conversion/ShapeToStandard/*.cpp", - "lib/Conversion/ShapeToStandard/*.h", ]), hdrs = ["include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h"], includes = ["include"], @@ -4645,7 +4607,6 @@ cc_library( name = "ShapeTransforms", srcs = glob([ "lib/Dialect/Shape/Transforms/*.cpp", - "lib/Dialect/Shape/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Shape/Analysis/ShapeMappingAnalysis.h", @@ -4728,7 +4689,6 @@ cc_library( srcs = glob( [ "lib/Dialect/ControlFlow/IR/*.cpp", - "lib/Dialect/ControlFlow/IR/*.h", ], ), hdrs = glob([ @@ -4774,13 +4734,10 @@ cc_library( srcs = glob( [ "lib/Dialect/Func/IR/*.cpp", - "lib/Dialect/Func/IR/*.h", - "lib/Dialect/Func/Utils/*.cpp", ], ), hdrs = glob([ "include/mlir/Dialect/Func/IR/*.h", - "include/mlir/Dialect/Func/Utils/*.h", ]), includes = ["include"], deps = [ @@ -4931,7 +4888,6 @@ cc_library( name = "FuncTransforms", srcs = glob([ "lib/Dialect/Func/Transforms/*.cpp", - "lib/Dialect/Func/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Func/Transforms/*.h"]), includes = ["include"], @@ -5068,7 +5024,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Vector/Transforms/*.cpp", - "lib/Dialect/Vector/Transforms/*.h", ], ), hdrs = glob([ @@ -5135,7 +5090,6 @@ cc_library( name = "Support", srcs = glob([ "lib/Support/*.cpp", - "lib/Support/*.h", ]), hdrs = glob(["include/mlir/Support/*.h"]), includes = ["include"], @@ -5149,11 +5103,8 @@ cc_library( name = "Debug", srcs = glob([ "lib/Debug/*.cpp", - "lib/Debug/*.h", "lib/Debug/BreakpointManagers/*.cpp", - "lib/Debug/BreakpointManagers/*.h", "lib/Debug/Observers/*.cpp", - "lib/Debug/Observers/*.h", ]), hdrs = glob([ "include/mlir/Debug/*.h", @@ -5195,8 +5146,6 @@ cc_library( [ "lib/Tools/mlir-lsp-server/*.cpp", "lib/Tools/mlir-lsp-server/*.h", - "lib/Tools/mlir-lsp-server/lsp/*.cpp", - "lib/Tools/mlir-lsp-server/lsp/*.h", ], ), hdrs = glob( @@ -5270,8 +5219,6 @@ cc_library( name = "BytecodeReader", srcs = glob([ "lib/Bytecode/Reader/*.cpp", - "lib/Bytecode/Reader/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5291,7 +5238,6 @@ cc_library( srcs = glob([ "lib/Bytecode/Writer/*.cpp", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5309,7 +5255,6 @@ cc_library( name = "Parser", srcs = glob([ "lib/Parser/*.cpp", - "lib/Parser/*.h", ]), hdrs = glob([ "include/mlir/Parser/*.h", @@ -5440,7 +5385,6 @@ cc_library( name = "LLVMIRTransforms", srcs = glob([ "lib/Dialect/LLVMIR/Transforms/*.cpp", - "lib/Dialect/LLVMIR/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/LLVMIR/Transforms/*.h"]), includes = ["include"], @@ -5593,7 +5537,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/IR/*.cpp", - "lib/Dialect/GPU/IR/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/IR/*.h"]), @@ -5687,7 +5630,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/Transforms/*.cpp", - "lib/Dialect/GPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/Transforms/*.h"]), @@ -5877,7 +5819,6 @@ cc_library( name = "GPUToNVVMTransforms", srcs = glob([ "lib/Conversion/GPUToNVVM/*.cpp", - "lib/Conversion/GPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToNVVM/*.h", @@ -5913,7 +5854,6 @@ cc_library( name = "AMDGPUToROCDL", srcs = glob([ "lib/Conversion/AMDGPUToROCDL/*.cpp", - "lib/Conversion/AMDGPUToROCDL/*.h", ]) + ["include/mlir/Conversion/GPUToROCDL/Runtimes.h"], hdrs = glob([ "include/mlir/Conversion/AMDGPUToROCDL/*.h", @@ -5938,7 +5878,6 @@ cc_library( name = "NVGPUToNVVM", srcs = glob([ "lib/Conversion/NVGPUToNVVM/*.cpp", - "lib/Conversion/NVGPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/NVGPUToNVVM/*.h", @@ -5966,7 +5905,6 @@ cc_library( name = "VectorToSPIRV", srcs = glob([ "lib/Conversion/VectorToSPIRV/*.cpp", - "lib/Conversion/VectorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSPIRV/*.h", @@ -6095,7 +6033,6 @@ cc_library( name = "GPUToSPIRV", srcs = glob([ "lib/Conversion/GPUToSPIRV/*.cpp", - "lib/Conversion/GPUToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToSPIRV/*.h", @@ -6665,7 +6602,6 @@ cc_library( name = "PDLDialect", srcs = glob([ "lib/Dialect/PDL/IR/*.cpp", - "lib/Dialect/PDL/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDL/IR/*.h", @@ -6740,7 +6676,6 @@ cc_library( name = "PDLInterpDialect", srcs = glob([ "lib/Dialect/PDLInterp/IR/*.cpp", - "lib/Dialect/PDLInterp/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDLInterp/IR/*.h", @@ -7038,7 +6973,6 @@ cc_library( srcs = glob( [ "lib/Dialect/SPIRV/Transforms/*.cpp", - "lib/Dialect/SPIRV/Transforms/*.h", ], exclude = ["lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp"], ), @@ -7077,7 +7011,6 @@ cc_library( name = "MathToSPIRV", srcs = glob([ "lib/Conversion/MathToSPIRV/*.cpp", - "lib/Conversion/MathToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToSPIRV/*.h", @@ -7104,7 +7037,6 @@ cc_library( name = "FuncToEmitC", srcs = glob([ "lib/Conversion/FuncToEmitC/*.cpp", - "lib/Conversion/FuncToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToEmitC/*.h", @@ -7130,7 +7062,6 @@ cc_library( name = "FuncToSPIRV", srcs = glob([ "lib/Conversion/FuncToSPIRV/*.cpp", - "lib/Conversion/FuncToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToSPIRV/*.h", @@ -7162,7 +7093,6 @@ cc_library( name = "TensorToLinalg", srcs = glob([ "lib/Conversion/TensorToLinalg/*.cpp", - "lib/Conversion/TensorToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToLinalg/*.h", @@ -7191,7 +7121,6 @@ cc_library( name = "TensorToSPIRV", srcs = glob([ "lib/Conversion/TensorToSPIRV/*.cpp", - "lib/Conversion/TensorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToSPIRV/*.h", @@ -7477,7 +7406,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Tensor/Transforms/*.cpp", - "lib/Dialect/Tensor/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Tensor/Transforms/*.h"]), @@ -7584,7 +7512,6 @@ cc_library( srcs = glob( include = [ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ], exclude = ["lib/Transforms/Utils/InliningUtils.cpp"], ), @@ -7923,7 +7850,6 @@ cc_library( name = "Transforms", srcs = glob([ "lib/Transforms/*.cpp", - "lib/Transforms/*.h", ]), hdrs = glob(["include/mlir/Transforms/*.h"]), includes = ["include"], @@ -8011,7 +7937,6 @@ cc_library( name = "SCFToSPIRV", srcs = glob([ "lib/Conversion/SCFToSPIRV/*.cpp", - "lib/Conversion/SCFToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/SCFToSPIRV/*.h", @@ -8261,7 +8186,6 @@ cc_library( name = "MemRefToEmitC", srcs = glob([ "lib/Conversion/MemRefToEmitC/*.cpp", - "lib/Conversion/MemRefToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToEmitC/*.h", @@ -8311,7 +8235,6 @@ cc_library( name = "MemRefToSPIRV", srcs = glob([ "lib/Conversion/MemRefToSPIRV/*.cpp", - "lib/Conversion/MemRefToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToSPIRV/*.h", @@ -8371,7 +8294,6 @@ cc_library( name = "ArithToArmSME", srcs = glob([ "lib/Conversion/ArithToArmSME/*.cpp", - "lib/Conversion/ArithToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToArmSME/*.h", @@ -8392,7 +8314,6 @@ cc_library( name = "ArithToEmitC", srcs = glob([ "lib/Conversion/ArithToEmitC/*.cpp", - "lib/Conversion/ArithToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToEmitC/*.h", @@ -8683,9 +8604,7 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", "lib/Analysis/*/*.cpp", - "lib/Analysis/*/*.h", ], ), hdrs = glob( @@ -8991,7 +8910,6 @@ cc_library( ":IR", ":LLVMDialect", ":OpenACCDialect", - ":OpenACCToLLVM", ":OpenMPCommon", ":Support", ":ToLLVMIRTranslation", @@ -9357,7 +9275,6 @@ cc_library( ":SCFToGPU", ":SCFTransformOps", ":SCFTransforms", - ":SDBM", ":SPIRVDialect", ":SPIRVPassIncGen", ":SPIRVTarget", @@ -9468,7 +9385,6 @@ cc_binary( "//mlir/test:TestTosaDialect", "//mlir/test:TestTransformDialect", "//mlir/test:TestTransforms", - "//mlir/test:TestTypeDialect", "//mlir/test:TestVector", "//mlir/test:TestVectorToSPIRV", ], @@ -10086,7 +10002,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/IR/*.cpp", - "lib/Dialect/OpenACC/IR/*.h", ], ), hdrs = glob( @@ -10140,7 +10055,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/Transforms/*.cpp", - "lib/Dialect/OpenACC/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/OpenACC/Transforms/*.h"]), @@ -10304,7 +10218,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenMP/IR/*.cpp", - "lib/Dialect/OpenMP/IR/*.h", ], ), hdrs = glob( @@ -10340,7 +10253,6 @@ cc_library( name = "OpenACCToSCF", srcs = glob([ "lib/Conversion/OpenACCToSCF/*.cpp", - "lib/Conversion/OpenACCToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenACCToSCF/*.h", @@ -10360,35 +10272,10 @@ cc_library( ], ) -cc_library( - name = "OpenACCToLLVM", - srcs = glob([ - "lib/Conversion/OpenACCToLLVM/*.cpp", - "lib/Conversion/OpenACCToLLVM/*.h", - ]), - hdrs = glob([ - "include/mlir/Conversion/OpenACCToLLVM/*.h", - ]), - includes = ["include"], - deps = [ - ":ConversionPassIncGen", - ":FuncDialect", - ":IR", - ":LLVMCommonConversion", - ":LLVMDialect", - ":OpenACCDialect", - ":Pass", - ":Transforms", - "//llvm:Core", - "//llvm:Support", - ], -) - cc_library( name = "OpenMPToLLVM", srcs = glob([ "lib/Conversion/OpenMPToLLVM/*.cpp", - "lib/Conversion/OpenMPToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenMPToLLVM/*.h", @@ -10598,7 +10485,6 @@ cc_library( name = "IndexToLLVM", srcs = glob([ "lib/Conversion/IndexToLLVM/*.cpp", - "lib/Conversion/IndexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToLLVM/*.h", @@ -10624,7 +10510,6 @@ cc_library( name = "IndexToSPIRV", srcs = glob([ "lib/Conversion/IndexToSPIRV/*.cpp", - "lib/Conversion/IndexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToSPIRV/*.h", @@ -11015,7 +10900,6 @@ cc_library( name = "LinalgToStandard", srcs = glob([ "lib/Conversion/LinalgToStandard/*.cpp", - "lib/Conversion/LinalgToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/LinalgToStandard/*.h", @@ -11161,7 +11045,6 @@ cc_library( name = "LinalgUtils", srcs = glob([ "lib/Dialect/Linalg/Utils/*.cpp", - "lib/Dialect/Linalg/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Linalg/Utils/*.h", @@ -11190,7 +11073,6 @@ cc_library( name = "LinalgTransforms", srcs = glob([ "lib/Dialect/Linalg/Transforms/*.cpp", - "lib/Dialect/Linalg/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Linalg/Passes.h", @@ -11542,7 +11424,6 @@ cc_library( name = "VectorToLLVM", srcs = glob([ "lib/Conversion/VectorToLLVM/*.cpp", - "lib/Conversion/VectorToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToLLVM/*.h", @@ -11584,7 +11465,6 @@ cc_library( name = "VectorToArmSME", srcs = glob([ "lib/Conversion/VectorToArmSME/*.cpp", - "lib/Conversion/VectorToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToArmSME/*.h", @@ -11605,7 +11485,6 @@ cc_library( name = "VectorToGPU", srcs = glob([ "lib/Conversion/VectorToGPU/*.cpp", - "lib/Conversion/VectorToGPU/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToGPU/*.h", @@ -11642,7 +11521,6 @@ cc_library( name = "VectorToSCF", srcs = glob([ "lib/Conversion/VectorToSCF/*.cpp", - "lib/Conversion/VectorToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSCF/*.h", @@ -11792,7 +11670,6 @@ cc_library( name = "TosaDialect", srcs = glob([ "lib/Dialect/Tosa/IR/*.cpp", - "lib/Dialect/Tosa/IR/*.h", "lib/Dialect/Tosa/Utils/*.cpp", "lib/Dialect/Tosa/Transforms/*.cpp", ]), @@ -11832,7 +11709,6 @@ cc_library( name = "TosaToArith", srcs = glob([ "lib/Conversion/TosaToArith/*.cpp", - "lib/Conversion/TosaToArith/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToArith/*.h", @@ -11856,7 +11732,6 @@ cc_library( name = "TosaToLinalg", srcs = glob([ "lib/Conversion/TosaToLinalg/*.cpp", - "lib/Conversion/TosaToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToLinalg/*.h", @@ -11889,7 +11764,6 @@ cc_library( name = "TosaToMLProgram", srcs = glob([ "lib/Conversion/TosaToMLProgram/*.cpp", - "lib/Conversion/TosaToMLProgram/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToMLProgram/*.h", @@ -11913,7 +11787,6 @@ cc_library( name = "TosaToSCF", srcs = glob([ "lib/Conversion/TosaToSCF/*.cpp", - "lib/Conversion/TosaToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToSCF/*.h", @@ -11938,7 +11811,6 @@ cc_library( name = "TosaToTensor", srcs = glob([ "lib/Conversion/TosaToTensor/*.cpp", - "lib/Conversion/TosaToTensor/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToTensor/*.h", @@ -12443,7 +12315,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Complex/IR/*.cpp", - "lib/Dialect/Complex/IR/*.h", ], ), hdrs = ["include/mlir/Dialect/Complex/IR/Complex.h"], @@ -12466,7 +12337,6 @@ cc_library( name = "ComplexToLLVM", srcs = glob([ "lib/Conversion/ComplexToLLVM/*.cpp", - "lib/Conversion/ComplexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLLVM/*.h", @@ -12494,7 +12364,6 @@ cc_library( name = "ComplexToLibm", srcs = glob([ "lib/Conversion/ComplexToLibm/*.cpp", - "lib/Conversion/ComplexToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLibm/*.h", @@ -12518,7 +12387,6 @@ cc_library( name = "ComplexToSPIRV", srcs = glob([ "lib/Conversion/ComplexToSPIRV/*.cpp", - "lib/Conversion/ComplexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToSPIRV/*.h", @@ -12543,7 +12411,6 @@ cc_library( name = "ComplexToStandard", srcs = glob([ "lib/Conversion/ComplexToStandard/*.cpp", - "lib/Conversion/ComplexToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToStandard/*.h", @@ -12760,7 +12627,6 @@ cc_library( name = "ArithTransforms", srcs = glob([ "lib/Dialect/Arith/Transforms/*.cpp", - "lib/Dialect/Arith/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Arith/Transforms/*.h", @@ -12882,7 +12748,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Math/IR/*.cpp", - "lib/Dialect/Math/IR/*.h", ], ), hdrs = [ @@ -12909,7 +12774,6 @@ cc_library( name = "MathTransforms", srcs = glob([ "lib/Dialect/Math/Transforms/*.cpp", - "lib/Dialect/Math/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Math/Transforms/*.h"]), includes = ["include"], @@ -12933,7 +12797,6 @@ cc_library( name = "MathToLibm", srcs = glob([ "lib/Conversion/MathToLibm/*.cpp", - "lib/Conversion/MathToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToLibm/*.h", @@ -13024,7 +12887,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/IR/*.cpp", - "lib/Dialect/MemRef/IR/*.h", ], ), hdrs = [ @@ -13102,7 +12964,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/Transforms/*.cpp", - "lib/Dialect/MemRef/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/MemRef/Transforms/*.h"]), @@ -13730,7 +13591,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Bufferization/Transforms/*.cpp", - "lib/Dialect/Bufferization/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Bufferization/Transforms/*.h"]), @@ -13922,7 +13782,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/ODS/*.cpp", - "lib/Tools/PDLL/ODS/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/ODS/*.h"]), @@ -13956,7 +13815,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/CodeGen/*.cpp", - "lib/Tools/PDLL/CodeGen/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/CodeGen/*.h"]), @@ -14117,7 +13975,6 @@ cc_library( name = "UBToLLVM", srcs = glob([ "lib/Conversion/UBToLLVM/*.cpp", - "lib/Conversion/UBToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToLLVM/*.h", @@ -14138,7 +13995,6 @@ cc_library( name = "UBToSPIRV", srcs = glob([ "lib/Conversion/UBToSPIRV/*.cpp", - "lib/Conversion/UBToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToSPIRV/*.h", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index df2392fd8c6e..57b29eb46e08 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -818,7 +818,6 @@ cc_library( cc_library( name = "TestMesh", srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]), - hdrs = glob(["lib/Dialect/Mesh/**/*.h"]), includes = ["lib/Dialect/Test"], deps = [ ":TestDialect", @@ -1046,18 +1045,6 @@ cc_library( ], ) -cc_library( - name = "TestTypeDialect", - srcs = glob([ - "lib/Dialect/LLVMIR/*.cpp", - ]), - deps = [ - ":TestDialect", - "//mlir:IR", - "//mlir:LLVMDialect", - ], -) - cc_library( name = "TestTosaDialect", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index c6b630230bb3..7172beb4de9a 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -16,7 +16,6 @@ cc_test( size = "small", srcs = glob([ "Debug/*.cpp", - "Debug/*.h", ]), deps = [ "//llvm:Support", @@ -35,7 +34,6 @@ cc_test( size = "small", srcs = glob([ "IR/*.cpp", - "IR/*.h", ]), deps = [ "//llvm:Support", @@ -57,7 +55,6 @@ cc_test( size = "small", srcs = glob([ "Interfaces/*.cpp", - "Interfaces/*.h", ]), deps = [ "//llvm:Support", @@ -82,7 +79,6 @@ cc_test( size = "small", srcs = glob([ "Support/*.cpp", - "Support/*.h", ]), deps = [ "//llvm:Support", @@ -98,7 +94,6 @@ cc_test( size = "small", srcs = glob([ "Pass/*.cpp", - "Pass/*.h", ]), deps = [ "//llvm:Support", @@ -118,7 +113,6 @@ cc_test( size = "small", srcs = glob([ "Rewrite/*.cpp", - "Rewrite/*.h", ]), deps = [ "//mlir:IR", @@ -134,7 +128,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/*.cpp", - "Dialect/*.h", ]), deps = [ "//llvm:Support", @@ -149,7 +142,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/MemRef/*.cpp", - "Dialect/MemRef/*.h", ]), deps = [ "//llvm:TestingSupport", @@ -161,27 +153,11 @@ cc_test( ], ) -cc_test( - name = "quantops_tests", - size = "small", - srcs = glob([ - "Dialect/Quant/*.cpp", - "Dialect/Quant/*.h", - ]), - deps = [ - "//llvm:TestingSupport", - "//mlir:QuantOps", - "//mlir:Transforms", - "//third-party/unittest:gtest_main", - ], -) - cc_test( name = "scf_tests", size = "small", srcs = glob([ "Dialect/SCF/*.cpp", - "Dialect/SCF/*.h", ]), deps = [ "//mlir:ArithDialect", @@ -200,7 +176,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SparseTensor/*.cpp", - "Dialect/SparseTensor/*.h", ]), deps = [ "//llvm:Support", @@ -218,7 +193,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SPIRV/*.cpp", - "Dialect/SPIRV/*.h", ]), deps = [ "//llvm:Support", @@ -238,7 +212,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Transform/*.cpp", - "Dialect/Transform/*.h", ]), deps = [ "//llvm:Support", @@ -264,7 +237,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Utils/*.cpp", - "Dialect/Utils/*.h", ]), deps = [ "//llvm:Support", @@ -317,7 +289,6 @@ cc_test( size = "small", srcs = glob([ "TableGen/*.cpp", - "TableGen/*.h", ]) + [ "TableGen/EnumsGenTest.cpp.inc", "TableGen/EnumsGenTest.h.inc", @@ -343,7 +314,6 @@ cc_test( size = "small", srcs = glob([ "Transforms/*.cpp", - "Transforms/*.h", ]), deps = [ "//mlir:AffineAnalysis", @@ -363,8 +333,6 @@ cc_test( name = "analysis_tests", size = "small", srcs = glob([ - "Analysis/*.cpp", - "Analysis/*.h", "Analysis/*/*.cpp", "Analysis/*/*.h", ]), @@ -387,9 +355,6 @@ cc_test( size = "small", srcs = glob([ "Bytecode/*.cpp", - "Bytecode/*.h", - "Bytecode/*/*.cpp", - "Bytecode/*/*.h", ]), deps = [ "//llvm:Support", @@ -408,10 +373,7 @@ cc_test( name = "conversion_tests", size = "small", srcs = glob([ - "Conversion/*.cpp", - "Conversion/*.h", "Conversion/*/*.cpp", - "Conversion/*/*.h", ]), deps = [ "//mlir:ArithDialect", -- GitLab From 5184e6ad69b0ca69dfba6fb0982a675c595f49a2 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 13:00:32 -0400 Subject: [PATCH 187/933] Removing accidental code from 527a624205748814dd9309eda7ee308b40b2359a --- clang/test/C/C11/n1365.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c index 3f769faa365d..d60bb546b29a 100644 --- a/clang/test/C/C11/n1365.c +++ b/clang/test/C/C11/n1365.c @@ -23,7 +23,7 @@ void func(void) { // CHECK-NEXT: FloatingLiteral // Ensure that a cast removes the extra precision. - _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + _Static_assert(123.0F * 2.0F == 246.0F, ""); // CHECK: StaticAssertDecl // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' -- GitLab From bd493756fa51e538575fc320aae50d75394f0567 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 17:01:39 +0000 Subject: [PATCH 188/933] [llvm-exegesis] Refactor parent code to separate function (#86232) This patch refactors the parent code to a separate function in the subprocess executor to make the code more clear and easy to follow. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d8da7bd86e3 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,7 +299,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -348,14 +309,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From d2f8ba7d6dc7251815f1431cf8715053576615f4 Mon Sep 17 00:00:00 2001 From: Sacha Coppey Date: Fri, 22 Mar 2024 18:08:13 +0100 Subject: [PATCH 189/933] [RISCV][NFC] Add generateMCInstSeq in RISCVMatInt (#84462) This allows to avoid duplicating the code handling the instructions outputted by `generateInstSeq` when emitting `MCInst`s. --- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 31 ++------------- .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp | 38 +++++++++++++++++++ .../Target/RISCV/MCTargetDesc/RISCVMatInt.h | 5 +++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 1779959324da..cb2ba52390e2 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3081,34 +3081,11 @@ void RISCVAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) { void RISCVAsmParser::emitLoadImm(MCRegister DestReg, int64_t Value, MCStreamer &Out) { - RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Value, getSTI()); - - MCRegister SrcReg = RISCV::X0; - for (const RISCVMatInt::Inst &Inst : Seq) { - switch (Inst.getOpndKind()) { - case RISCVMatInt::Imm: - emitToStreamer(Out, - MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addImm(Inst.getImm())); - break; - case RISCVMatInt::RegX0: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - RISCV::X0)); - break; - case RISCVMatInt::RegReg: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - SrcReg)); - break; - case RISCVMatInt::RegImm: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addImm( - Inst.getImm())); - break; - } + SmallVector Seq; + RISCVMatInt::generateMCInstSeq(Value, getSTI(), DestReg, Seq); - // Only the first instruction has X0 as its source. - SrcReg = DestReg; + for (MCInst &Inst : Seq) { + emitToStreamer(Out, Inst); } } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp index 4358a5b878e6..c3bae152993e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp @@ -9,6 +9,7 @@ #include "RISCVMatInt.h" #include "MCTargetDesc/RISCVMCTargetDesc.h" #include "llvm/ADT/APInt.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/MathExtras.h" using namespace llvm; @@ -436,6 +437,43 @@ InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI) { return Res; } +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts) { + RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI); + + MCRegister SrcReg = RISCV::X0; + for (RISCVMatInt::Inst &Inst : Seq) { + switch (Inst.getOpndKind()) { + case RISCVMatInt::Imm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addImm(Inst.getImm())); + break; + case RISCVMatInt::RegX0: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(RISCV::X0)); + break; + case RISCVMatInt::RegReg: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(SrcReg)); + break; + case RISCVMatInt::RegImm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addImm(Inst.getImm())); + break; + } + + // Only the first instruction has X0 as its source. + SrcReg = DestReg; + } +} + InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc) { int64_t LoVal = SignExtend64<32>(Val); diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h index 780f685463f3..e87e0f325647 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_MATINT_H #include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCRegister.h" #include "llvm/MC/MCSubtargetInfo.h" #include @@ -48,6 +49,10 @@ using InstSeq = SmallVector; // instruction selection. InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI); +// Helper to generate the generateInstSeq instruction sequence using MCInsts +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts); + // Helper to generate an instruction sequence that can materialize the given // immediate value into a register using an additional temporary register. This // handles cases where the constant can be generated by (ADD (SLLI X, C), X) or -- GitLab From c3a41aac5f32475b9a0499e6e888e713763566dc Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:25:23 -0700 Subject: [PATCH 190/933] Revert "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit bd493756fa51e538575fc320aae50d75394f0567. Causes build failures on non-X86 platforms. https://lab.llvm.org/buildbot/#/changes/128363 --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++---------- 1 file changed, 53 insertions(+), 63 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 6d8da7bd86e3..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,20 +278,59 @@ private: return FD; } - Error - runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, - SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + prepareAndRunBenchmark(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = - ET.createCounter(CounterName, State, ValidationCounters, ChildPID); + auto CounterOrError = ET.createCounter( + CounterName, State, ValidationCounters, ParentOrChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); + close(PipeFiles[0]); + // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -299,7 +338,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -309,14 +348,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); + sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); if (SendError) return SendError; @@ -356,7 +395,8 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { + if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == + -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -382,56 +422,6 @@ private: return make_error(ChildSignalInfo.si_signo); } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - runChildSubprocess(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - - // Close the read end of the pipe as we only need to write to the subprocess - // from the parent process. - close(PipeFiles[0]); - return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, - CounterValues, ValidationCounters, - ValidationCounterValues); - } - void disableCoreDumps() const { struct rlimit rlim; @@ -439,8 +429,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void runChildSubprocess(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void prepareAndRunBenchmark(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From 36a6afdd2c7fa02548260ebe4c993b705c6e6e38 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:26:24 -0700 Subject: [PATCH 191/933] Reland "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit c3a41aac5f32475b9a0499e6e888e713763566dc. This relands commit bd493756fa51e538575fc320aae50d75394f0567. Apparently I forgot to update a couple values, so this change failed on every builder that builds those sections (should be every Linux platform) rather than something architecture specific like originally thought. I swore I updated the values and ran check-llvm before merging. Wondering If I forgot to push those changes... --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 124 ++++++++++-------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d5f0286fb9c 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,30 +299,30 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (waitpid(ParentOrChildPID, NULL, 0) == -1) { + if (waitpid(ChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; int ChildStatus; - if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { + if (waitpid(ChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -405,13 +365,13 @@ private: // handlers to run, and calling SIGTERM would mean that ptrace will force // it to block in the signal-delivery-stop for the SIGSEGV/other signals, // and upon exit. - if (kill(ParentOrChildPID, SIGKILL) == -1) + if (kill(ChildPID, SIGKILL) == -1) return make_error("Failed to kill child benchmarking proces: " + Twine(strerror(errno))); // Wait for the process to exit so that there are no zombie processes left // around. - if (waitpid(ParentOrChildPID, NULL, 0) == -1) + if (waitpid(ChildPID, NULL, 0) == -1) return make_error("Failed to wait for process to die: " + Twine(strerror(errno))); @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From bbcfe6f4311af8cf6095a5bc5937fa68a87b4289 Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:37:39 -0500 Subject: [PATCH 192/933] [mlir][transform] Emit error message with `emitSilenceableFailure` (#86146) The previous implementation used a `notifyMatchFailure` to emit failure message inappropriately and then used the `emitDefaultSilenceableFailure`. This patch changes this to use the more appropriate `emitSilenceableFailure` with error message. Additionally a failure test has been added. --- .../TransformOps/LinalgTransformOps.cpp | 14 ++++---- .../Dialect/Linalg/flatten-unsupported.mlir | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 mlir/test/Dialect/Linalg/flatten-unsupported.mlir diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index ecf998312482..88819cd96435 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -3269,22 +3269,24 @@ DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::ApplyToEachResultList &results, transform::TransformState &state) { rewriter.setInsertionPoint(target); - if (!isElementwise(target)) { - failed(rewriter.notifyMatchFailure( - target, "only elementwise flattening is supported")); - return emitDefaultSilenceableFailure(target); - } + if (!isElementwise(target)) + return mlir::emitSilenceableFailure(target->getLoc()) + << "only elementwise flattening is supported"; + // If rank <= 1, do nothing if (target.getNumLoops() <= 1) { results.push_back(target); return DiagnosedSilenceableFailure::success(); } + + // Attempt to flatten all dims to one. ReassociationIndices reassociation(target.getNumLoops()); std::iota(reassociation.begin(), reassociation.end(), 0); auto maybeFlattened = collapseOpIterationDims(target, reassociation, rewriter); if (failed(maybeFlattened)) - return emitDefaultSilenceableFailure(target); + return mlir::emitSilenceableFailure(target->getLoc()) + << "attempted to flatten, but failed"; results.push_back(maybeFlattened->collapsedOp); rewriter.replaceOp(target, maybeFlattened->results); return DiagnosedSilenceableFailure::success(); diff --git a/mlir/test/Dialect/Linalg/flatten-unsupported.mlir b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir new file mode 100644 index 000000000000..499db4cfb329 --- /dev/null +++ b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir @@ -0,0 +1,33 @@ +// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics + +func.func @non_elementwise(%arg0: memref<2x3xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) { + // expected-error @below {{only elementwise flattening is supported}} + linalg.matmul ins(%arg0, %arg1 : memref<2x3xf32>, memref<3x4xf32>) outs(%arg2: memref<2x4xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} + +// ----- + +func.func @unsupported_memref(%arg0: memref<32x7xf32, strided<[7, 2]>>, %arg1: memref<32x7xf32, strided<[7, 2]>>, %arg2: memref<32x7xf32, strided<[7, 2]>>) { + // expected-error @below {{attempted to flatten, but failed}} + linalg.map {arith.addf} ins(%arg0, %arg1: memref<32x7xf32, strided<[7, 2]>>, memref<32x7xf32, strided<[7, 2]>>) outs(%arg2: memref<32x7xf32, strided<[7, 2]>>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} -- GitLab From cd8286a667d568c4319b09baa63ba899e3101a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danny=20M=C3=B6sch?= Date: Fri, 22 Mar 2024 18:51:44 +0100 Subject: [PATCH 193/933] [GitHub] Allow shortcut for "introductory issue" and request linking to issue in PR (#84635) The answer to many requests in issues to be assigned to users is often "just create a pull request". That's in contradiction to the "introductory issue" instructions posted by the GitHub bot. This change updates the instructions, mentioning the shortcut of "just creating a PR". Moreover, it now explains linking PRs to issues in order to close them automatically upon merge. --- llvm/utils/git/github-automation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py index b21f14eca445..42a658cefac3 100755 --- a/llvm/utils/git/github-automation.py +++ b/llvm/utils/git/github-automation.py @@ -24,12 +24,13 @@ Hi! This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are: -1. In the comments of the issue, request for it to be assigned to you. -2. Fix the issue locally. -3. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. -4. Create a Git commit. -5. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. -6. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). +1. Check that no other contributor has already been assigned to this issue. If you believe that no one is actually working on it despite an assignment, ping the person. After one week without a response, the assignee may be changed. +1. In the comments of this issue, request for it to be assigned to you, or just create a [pull request](https://github.com/llvm/llvm-project/pulls) after following the steps below. [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. +1. Fix the issue locally. +1. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. +1. Create a Git commit. +1. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. +1. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. If you have any further questions about this issue, don't hesitate to ask via a comment in the thread below. """ -- GitLab From fb329f18445cb33d242cc500ca618d03674b22ad Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:15:45 -0700 Subject: [PATCH 194/933] [Target] Move SubRegIdxRanges from MCSubtargetInfo to TargetInfo. (#86245) I'm planning to add HwMode support to SubRegIdxRanges for RISC-V GPR pairs. The MC layer is currently unaware of the HwMode for registers and I'd like to keep it that way. This information is not used by the MC layer so I think it is safe to move it. --- .../include/llvm/CodeGen/TargetRegisterInfo.h | 31 ++++++++++++---- llvm/include/llvm/MC/MCRegisterInfo.h | 21 ----------- llvm/lib/CodeGen/TargetRegisterInfo.cpp | 36 +++++++++++-------- llvm/lib/MC/MCRegisterInfo.cpp | 12 ------- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 4 +-- llvm/unittests/CodeGen/MFCommon.inc | 5 +-- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 29 +++++++-------- 7 files changed, 65 insertions(+), 73 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 117d3f718297..33c4c745c341 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -243,9 +243,20 @@ public: unsigned RegSize, SpillSize, SpillAlignment; unsigned VTListOffset; }; + + /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg + /// index, -1 in any being invalid. + struct SubRegCoveredBits { + uint16_t Offset; + uint16_t Size; + }; + private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. + const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered + // bit ranges array. + // Pointer to array of lane masks, one per sub-reg index. const LaneBitmask *SubRegIndexLaneMasks; @@ -256,12 +267,10 @@ private: unsigned HwMode; protected: - TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, - regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask CoveringLanes, + TargetRegisterInfo(const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, + const LaneBitmask *SRILaneMasks, LaneBitmask CoveringLanes, const RegClassInfo *const RCIs, const MVT::SimpleValueType *const RCVTLists, unsigned Mode = 0); @@ -382,6 +391,16 @@ public: return SubRegIndexNames[SubIdx-1]; } + /// Get the size of the bit range covered by a sub-register index. + /// If the index isn't continuous, return the sum of the sizes of its parts. + /// If the index is used to access subregisters of different sizes, return -1. + unsigned getSubRegIdxSize(unsigned Idx) const; + + /// Get the offset of the bit range covered by a sub-register index. + /// If an Offset doesn't make sense (the index isn't continuous, or is used to + /// access sub-registers at different offsets), return -1. + unsigned getSubRegIdxOffset(unsigned Idx) const; + /// Return a bitmask representing the parts of a register that are covered by /// SubIdx \see LaneBitmask. /// diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index fb4d11ec1d4d..c648ef20fa84 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -153,13 +153,6 @@ public: bool operator<(DwarfLLVMRegPair RHS) const { return FromReg < RHS.FromReg; } }; - /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg - /// index, -1 in any being invalid. - struct SubRegCoveredBits { - uint16_t Offset; - uint16_t Size; - }; - private: const MCRegisterDesc *Desc; // Pointer to the descriptor array unsigned NumRegs; // Number of entries in the array @@ -176,8 +169,6 @@ private: const char *RegClassStrings; // Pointer to the class strings. const uint16_t *SubRegIndices; // Pointer to the subreg lookup // array. - const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered - // bit ranges array. unsigned NumSubRegIndices; // Number of subreg indices. const uint16_t *RegEncodingTable; // Pointer to array of register // encodings. @@ -278,7 +269,6 @@ public: const int16_t *DL, const LaneBitmask *RUMS, const char *Strings, const char *ClassStrings, const uint16_t *SubIndices, unsigned NumIndices, - const SubRegCoveredBits *SubIdxRanges, const uint16_t *RET) { Desc = D; NumRegs = NR; @@ -294,7 +284,6 @@ public: NumRegUnits = NRU; SubRegIndices = SubIndices; NumSubRegIndices = NumIndices; - SubRegIdxRanges = SubIdxRanges; RegEncodingTable = RET; // Initialize DWARF register mapping variables @@ -387,16 +376,6 @@ public: /// otherwise. unsigned getSubRegIndex(MCRegister RegNo, MCRegister SubRegNo) const; - /// Get the size of the bit range covered by a sub-register index. - /// If the index isn't continuous, return the sum of the sizes of its parts. - /// If the index is used to access subregisters of different sizes, return -1. - unsigned getSubRegIdxSize(unsigned Idx) const; - - /// Get the offset of the bit range covered by a sub-register index. - /// If an Offset doesn't make sense (the index isn't continuous, or is used to - /// access sub-registers at different offsets), return -1. - unsigned getSubRegIdxOffset(unsigned Idx) const; - /// Return the human-readable symbolic target-specific name for the /// specified physical register. const char *getName(MCRegister RegNo) const { diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp index c9503fcb77bb..4120c74c23b1 100644 --- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp +++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -50,20 +50,16 @@ static cl::opt "high compile time cost in global splitting."), cl::init(5000)); -TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask SRICoveringLanes, - const RegClassInfo *const RCIs, - const MVT::SimpleValueType *const RCVTLists, - unsigned Mode) - : InfoDesc(ID), SubRegIndexNames(SRINames), - SubRegIndexLaneMasks(SRILaneMasks), - RegClassBegin(RCB), RegClassEnd(RCE), - CoveringLanes(SRICoveringLanes), - RCInfos(RCIs), RCVTLists(RCVTLists), HwMode(Mode) { -} +TargetRegisterInfo::TargetRegisterInfo( + const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, const LaneBitmask *SRILaneMasks, + LaneBitmask SRICoveringLanes, const RegClassInfo *const RCIs, + const MVT::SimpleValueType *const RCVTLists, unsigned Mode) + : InfoDesc(ID), SubRegIndexNames(SRINames), SubRegIdxRanges(SubIdxRanges), + SubRegIndexLaneMasks(SRILaneMasks), RegClassBegin(RCB), RegClassEnd(RCE), + CoveringLanes(SRICoveringLanes), RCInfos(RCIs), RCVTLists(RCVTLists), + HwMode(Mode) {} TargetRegisterInfo::~TargetRegisterInfo() = default; @@ -596,6 +592,18 @@ bool TargetRegisterInfo::getCoveringSubRegIndexes( return BestIdx; } +unsigned TargetRegisterInfo::getSubRegIdxSize(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Size; +} + +unsigned TargetRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Offset; +} + Register TargetRegisterInfo::lookThruCopyLike(Register SrcReg, const MachineRegisterInfo *MRI) const { diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp index a2c1737e2964..334655616d8d 100644 --- a/llvm/lib/MC/MCRegisterInfo.cpp +++ b/llvm/lib/MC/MCRegisterInfo.cpp @@ -57,18 +57,6 @@ unsigned MCRegisterInfo::getSubRegIndex(MCRegister Reg, return 0; } -unsigned MCRegisterInfo::getSubRegIdxSize(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Size; -} - -unsigned MCRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Offset; -} - int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const { const DwarfLLVMRegPair *M = isEH ? EHL2DwarfRegs : L2DwarfRegs; unsigned Size = isEH ? EHL2DwarfRegsSize : L2DwarfRegsSize; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 79a7d1cf66c4..245731ad5fc7 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -363,8 +363,8 @@ SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) for (auto &Row : SubRegFromChannelTable) Row.fill(AMDGPU::NoSubRegister); for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { - unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; - unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; + unsigned Width = getSubRegIdxSize(Idx) / 32; + unsigned Offset = getSubRegIdxOffset(Idx) / 32; assert(Width < SubRegFromChannelTableWidthMap.size()); Width = SubRegFromChannelTableWidthMap[Width]; if (Width == 0) diff --git a/llvm/unittests/CodeGen/MFCommon.inc b/llvm/unittests/CodeGen/MFCommon.inc index 7de7eabdd1f6..1997e8052297 100644 --- a/llvm/unittests/CodeGen/MFCommon.inc +++ b/llvm/unittests/CodeGen/MFCommon.inc @@ -23,9 +23,10 @@ class BogusRegisterInfo : public TargetRegisterInfo { public: BogusRegisterInfo() : TargetRegisterInfo(nullptr, BogusRegisterClasses, BogusRegisterClasses, - nullptr, nullptr, LaneBitmask(~0u), nullptr, nullptr) { + nullptr, nullptr, nullptr, LaneBitmask(~0u), nullptr, + nullptr) { InitMCRegisterInfo(nullptr, 0, 0, 0, nullptr, 0, nullptr, 0, nullptr, - nullptr, nullptr, nullptr, nullptr, 0, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, 0, nullptr); } const MCPhysReg * diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index d074e31c6245..c4fc1930488c 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -955,16 +955,6 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, SubRegIdxSeqs.emit(OS, printSubRegIndex); OS << "};\n\n"; - // Emit the table of sub-register index sizes. - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[] = {\n"; - OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; - for (const auto &Idx : SubRegIndices) { - OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " - << Idx.getName() << "\n"; - } - OS << "};\n\n"; - // Emit the string table. RegStrings.layout(); RegStrings.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName + @@ -1101,8 +1091,7 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, << TargetName << "LaneMaskLists, " << TargetName << "RegStrings, " << TargetName << "RegClassStrings, " << TargetName << "SubRegIdxLists, " << (std::distance(SubRegIndices.begin(), SubRegIndices.end()) + 1) << ",\n" - << TargetName << "SubRegIdxRanges, " << TargetName - << "RegEncodingTable);\n\n"; + << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, false); @@ -1253,6 +1242,16 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, } OS << "\" };\n\n"; + // Emit the table of sub-register index sizes. + OS << "static const TargetRegisterInfo::SubRegCoveredBits " + "SubRegIdxRangeTable[] = {\n"; + OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; + for (const auto &Idx : SubRegIndices) { + OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " + << Idx.getName() << "\n"; + } + OS << "};\n\n"; + // Emit SubRegIndex lane masks, including 0. OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n " "LaneBitmask::getAll(),\n"; @@ -1634,8 +1633,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, OS << "extern const char " << TargetName << "RegClassStrings[];\n"; OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2];\n"; OS << "extern const uint16_t " << TargetName << "SubRegIdxLists[];\n"; - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[];\n"; OS << "extern const uint16_t " << TargetName << "RegEncodingTable[];\n"; EmitRegMappingTables(OS, Regs, true); @@ -1646,7 +1643,8 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, " unsigned PC, unsigned HwMode)\n" << " : TargetRegisterInfo(&" << TargetName << "RegInfoDesc" << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n" - << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" + << " SubRegIndexNameTable, SubRegIdxRangeTable, " + "SubRegIndexLaneMaskTable,\n" << " "; printMask(OS, RegBank.CoveringLanes); OS << ", RegClassInfos, VTLists, HwMode) {\n" @@ -1661,7 +1659,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, << " " << TargetName << "RegClassStrings,\n" << " " << TargetName << "SubRegIdxLists,\n" << " " << SubRegIndicesSize + 1 << ",\n" - << " " << TargetName << "SubRegIdxRanges,\n" << " " << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, true); -- GitLab From b1575f9082071702bd6aaa2600ce9fe011a091e9 Mon Sep 17 00:00:00 2001 From: Kevin Frei Date: Fri, 22 Mar 2024 11:22:09 -0700 Subject: [PATCH 195/933] Missed a null-ptr check in previous PR for Debuginfod testing (#86292) @GeorgeHuyubo noticed an unchecked shared pointer result in https://github.com/llvm/llvm-project/pull/85693/. This is the fix for that issue. Co-authored-by: Kevin Frei --- lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index 91b8b4a979e0..a9956aa9075f 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), dwp_file_data_sp, dwp_file_data_offset); - if (!ObjectFileELF::classof(dwp_obj_file.get())) + if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get())) return false; // The presence of a debug_cu_index section is the key identifying feature of // a DWP file. Make sure we don't fill in the section list on dwp_obj_file -- GitLab From 721f149596f27f3d4c5c28ec2a2fac33340fb876 Mon Sep 17 00:00:00 2001 From: Job Henandez Lara Date: Fri, 22 Mar 2024 11:38:51 -0700 Subject: [PATCH 196/933] Fix typo (#86319) Im working on the floating point fmaximum and fminimum functions right now in a different pr and I ran the individual tests by doing ``` ninja libc.test.src.math.smoke._test.__unit__ ``` --- libc/src/math/docs/add_math_function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md index f8bc8a3bdd8b..d1bca222b428 100644 --- a/libc/src/math/docs/add_math_function.md +++ b/libc/src/math/docs/add_math_function.md @@ -177,7 +177,7 @@ implementation (which is very often glibc). - Build and Run a specific unit test: ``` - $ ninja libc.test.src.math._test + $ ninja libc.test.src.math._test.__unit__ $ projects/libc/test/src/math/libc.test.src.math._test ``` -- GitLab From 83e96977cdb6041196366fc01e8abdca52cadb2e Mon Sep 17 00:00:00 2001 From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com> Date: Sat, 23 Mar 2024 00:09:53 +0530 Subject: [PATCH 197/933] [libc] Implement strfromd() and strfroml() (#86113) Follow up to #85438. Implements the functions `strfromd()` and `strfroml()` introduced in C23, and unifies the testing framework for `strfrom*()` functions. --- libc/config/linux/x86_64/entrypoints.txt | 2 + libc/spec/stdc.td | 2 + libc/src/stdlib/CMakeLists.txt | 20 ++ libc/src/stdlib/strfromd.cpp | 42 +++ libc/src/stdlib/strfromd.h | 21 ++ libc/src/stdlib/strfromf.h | 2 +- libc/src/stdlib/strfroml.cpp | 47 +++ libc/src/stdlib/strfroml.h | 21 ++ libc/test/src/stdlib/CMakeLists.txt | 31 ++ libc/test/src/stdlib/StrfromTest.h | 435 +++++++++++++++++++++++ libc/test/src/stdlib/strfromd_test.cpp | 13 + libc/test/src/stdlib/strfromf_test.cpp | 98 +---- libc/test/src/stdlib/strfroml_test.cpp | 13 + 13 files changed, 650 insertions(+), 97 deletions(-) create mode 100644 libc/src/stdlib/strfromd.cpp create mode 100644 libc/src/stdlib/strfromd.h create mode 100644 libc/src/stdlib/strfroml.cpp create mode 100644 libc/src/stdlib/strfroml.h create mode 100644 libc/test/src/stdlib/StrfromTest.h create mode 100644 libc/test/src/stdlib/strfromd_test.cpp create mode 100644 libc/test/src/stdlib/strfroml_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 8e1ab5cd65f0..8b1cd3fb1052 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -180,7 +180,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.srand + libc.src.stdlib.strfromd libc.src.stdlib.strfromf + libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 76010a4b4533..3e58e3b88645 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -962,6 +962,8 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"srand", RetValSpec, [ArgSpec]>, FunctionSpec<"strfromf", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfromd", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfroml", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, FunctionSpec<"strtof", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"strtod", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 22f7f990fb08..a2b16e73f353 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -62,6 +62,26 @@ add_entrypoint_object( .str_from_util ) +add_entrypoint_object( + strfromd + SRCS + strfromd.cpp + HDRS + strfromd.h + DEPENDS + .str_from_util +) + +add_entrypoint_object( + strfroml + SRCS + strfroml.cpp + HDRS + strfroml.h + DEPENDS + .str_from_util +) + add_header_library( str_from_util HDRS diff --git a/libc/src/stdlib/strfromd.cpp b/libc/src/stdlib/strfromd.cpp new file mode 100644 index 000000000000..1d02a7ad1124 --- /dev/null +++ b/libc/src/stdlib/strfromd.cpp @@ -0,0 +1,42 @@ +//===-- Implementation of strfromd ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfromd.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfromd, + (char *__restrict s, size_t n, const char *__restrict format, + double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfromd.h b/libc/src/stdlib/strfromd.h new file mode 100644 index 000000000000..d2c3fefb6300 --- /dev/null +++ b/libc/src/stdlib/strfromd.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfromd ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROMD_H +#define LLVM_LIBC_SRC_STDLIB_STRFROMD_H + +#include + +namespace LIBC_NAMESPACE { + +int strfromd(char *__restrict s, size_t n, const char *__restrict format, + double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMD_H diff --git a/libc/src/stdlib/strfromf.h b/libc/src/stdlib/strfromf.h index b551a58af05a..492c2c33cf08 100644 --- a/libc/src/stdlib/strfromf.h +++ b/libc/src/stdlib/strfromf.h @@ -18,4 +18,4 @@ int strfromf(char *__restrict s, size_t n, const char *__restrict format, } // namespace LIBC_NAMESPACE -#endif // LLVM_LIBC_SRC_STDLIB_STRTOF_H +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMF_H diff --git a/libc/src/stdlib/strfroml.cpp b/libc/src/stdlib/strfroml.cpp new file mode 100644 index 000000000000..6b8781e7c04e --- /dev/null +++ b/libc/src/stdlib/strfroml.cpp @@ -0,0 +1,47 @@ +//===-- Implementation of strfroml ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfroml.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfroml, + (char *__restrict s, size_t n, const char *__restrict format, + long double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + + // To ensure that the conversion function actually uses long double, + // the length modifier has to be set to LenghtModifier::L + section.length_modifier = printf_core::LengthModifier::L; + + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfroml.h b/libc/src/stdlib/strfroml.h new file mode 100644 index 000000000000..e99d035e4da6 --- /dev/null +++ b/libc/src/stdlib/strfroml.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfroml ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROML_H +#define LLVM_LIBC_SRC_STDLIB_STRFROML_H + +#include + +namespace LIBC_NAMESPACE { + +int strfroml(char *__restrict s, size_t n, const char *__restrict format, + long double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROML_H diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt index cb42bc56f51c..3ccc1cde9193 100644 --- a/libc/test/src/stdlib/CMakeLists.txt +++ b/libc/test/src/stdlib/CMakeLists.txt @@ -168,6 +168,14 @@ add_libc_test( .strtol_test_support ) +add_header_library( + strfrom_test_support + HDRS + StrfromTest.h + DEPENDS + libc.src.__support.CPP.type_traits +) + add_libc_test( strfromf_test SUITE @@ -175,9 +183,32 @@ add_libc_test( SRCS strfromf_test.cpp DEPENDS + .strfrom_test_support libc.src.stdlib.strfromf ) +add_libc_test( + strfromd_test + SUITE + libc-stdlib-tests + SRCS + strfromd_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfromd +) + +add_libc_test( + strfroml_test + SUITE + libc-stdlib-tests + SRCS + strfroml_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfroml +) + add_libc_test( abs_test SUITE diff --git a/libc/test/src/stdlib/StrfromTest.h b/libc/test/src/stdlib/StrfromTest.h new file mode 100644 index 000000000000..f695bbb335bd --- /dev/null +++ b/libc/test/src/stdlib/StrfromTest.h @@ -0,0 +1,435 @@ +//===-- A template class for testing strfrom functions ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/type_traits.h" +#include "test/UnitTest/Test.h" + +#define ASSERT_STREQ_LEN(actual_written, actual_str, expected_str) \ + EXPECT_EQ(actual_written, static_cast(sizeof(expected_str) - 1)); \ + EXPECT_STREQ(actual_str, expected_str); + +template +class StrfromTest : public LIBC_NAMESPACE::testing::Test { + + static const bool is_single_prec = + LIBC_NAMESPACE::cpp::is_same::value; + static const bool is_double_prec = + LIBC_NAMESPACE::cpp::is_same::value; + + using FunctionT = int (*)(char *, size_t, const char *, InputT fp); + +public: + void floatDecimalFormat(FunctionT func) { + if (is_single_prec) + floatDecimalSinglePrec(func); + else if (is_double_prec) + floatDecimalDoublePrec(func); + else + floatDecimalLongDoublePrec(func); + } + + void floatHexExpFormat(FunctionT func) { + if (is_single_prec) + floatHexExpSinglePrec(func); + else if (is_double_prec) + floatHexExpDoublePrec(func); + else + floatHexExpLongDoublePrec(func); + } + + void floatDecimalExpFormat(FunctionT func) { + if (is_single_prec) + floatDecimalExpSinglePrec(func); + else if (is_double_prec) + floatDecimalExpDoublePrec(func); + else + floatDecimalExpLongDoublePrec(func); + } + + void floatDecimalAutoFormat(FunctionT func) { + if (is_single_prec) + floatDecimalAutoSinglePrec(func); + else if (is_double_prec) + floatDecimalAutoDoublePrec(func); + else + floatDecimalAutoLongDoublePrec(func); + } + + void improperFormatString(FunctionT func) { + char buff[100]; + int written; + const bool is_long_double = !is_single_prec && !is_double_prec; + + written = func(buff, 37, "A simple string with no conversions.", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with no conversions."); + + written = + func(buff, 37, + "%A simple string with one conversion, should overwrite.", 1.0); + ASSERT_STREQ_LEN(written, buff, is_long_double ? "0X8P-3" : "0X1P+0"); + + written = func(buff, 74, + "A simple string with one conversion in %A " + "between, writes string as it is", + 1.0); + ASSERT_STREQ_LEN(written, buff, + "A simple string with one conversion in %A between, " + "writes string as it is"); + + written = func(buff, 36, "A simple string with one conversion", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with one conversion"); + + written = func(buff, 20, "%1f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "%1f"); + } + + void insufficentBufsize(FunctionT func) { + char buff[20]; + int written; + + written = func(buff, 5, "%f", 1234567890.0); + EXPECT_EQ(written, 17); + ASSERT_STREQ(buff, "1234"); + + written = func(buff, 5, "%.5f", 1.05); + EXPECT_EQ(written, 7); + ASSERT_STREQ(buff, "1.05"); + + written = func(buff, 0, "%g", 1.0); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "1.05"); // Make sure that buff has not changed + } + + void floatDecimalSinglePrec(FunctionT func) { + char buff[70]; + int written; + + written = func(buff, 16, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 20, "%f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1234567936.000000"); + + written = func(buff, 67, "%.3f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000"); + } + + void floatDecimalDoublePrec(FunctionT func) { + char buff[500]; + int written; + + written = func(buff, 99, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 99, "%F", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000"); + + written = func(buff, 99, "%f", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567"); + + written = func(buff, 99, "%f", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000"); + + written = func(buff, 99, "%f", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000"); + + written = func(buff, 499, "%f", 1e300); + ASSERT_STREQ_LEN(written, buff, + "100000000000000005250476025520442024870446858110815915491" + "585411551180245" + "798890819578637137508044786404370444383288387817694252323" + "536043057564479" + "218478670698284838720092657580373783023379478809005936895" + "323497079994508" + "111903896764088007465274278014249457925878882005684283811" + "566947219638686" + "5459400540160.000000"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + // Precision Tests + written = func(buff, 100, "%.2f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.99"); + + written = func(buff, 100, "%.1f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "10000000000000.0"); + + written = func(buff, 100, "%.5f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1.25000"); + + written = func(buff, 100, "%.0f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1"); + + written = func(buff, 100, "%.20f", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "0.00000000012340000000"); + } + + void floatDecimalLongDoublePrec(FunctionT func) { + char buff[45]; + int written; + + written = func(buff, 40, "%f", 1.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 10, "%.f", -2.5L); + ASSERT_STREQ_LEN(written, buff, "-2"); + } + + void floatHexExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 0, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + + written = func(buff, 20, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0x1.26580cp+30"); + + written = func(buff, 20, "%A", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0X1.26580CP+30"); + } + + void floatHexExpDoublePrec(FunctionT func) { + char buff[60]; + int written; + + written = func(buff, 10, "%a", 1.0); + ASSERT_STREQ_LEN(written, buff, "0x1p+0"); + + written = func(buff, 10, "%A", -1.0); + ASSERT_STREQ_LEN(written, buff, "-0X1P+0"); + + written = func(buff, 30, "%a", -0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "-0x1.abcdef12345p+0"); + + written = func(buff, 50, "%A", 0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "0X1.ABCDEF12345P+0"); + + written = func(buff, 10, "%a", 0.0); + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); + + written = func(buff, 40, "%a", 1.0e100); + ASSERT_STREQ_LEN(written, buff, "0x1.249ad2594c37dp+332"); + + written = func(buff, 30, "%a", 0.1); + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); + } + + void floatHexExpLongDoublePrec(FunctionT func) { + char buff[55]; + int written; + + written = func(buff, 50, "%a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.ccccccccccccccdp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4"); +#endif + + written = func(buff, 20, "%.1a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.dp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#endif + + written = func(buff, 50, "%a", 1.0e1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xf.38db1f9dd3dac05p+3318"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321"); +#endif + + written = func(buff, 50, "%a", 1.0e-1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x8.68a9188a89e1467p-3325"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322"); +#endif + + written = func(buff, 50, "%.1a", 0xf.fffffffffffffffp16380L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x1.0p+16384"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383"); +#endif + } + + void floatDecimalExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9e", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936e+09"); + + written = func(buff, 20, "%.9E", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936E+09"); + } + + void floatDecimalExpDoublePrec(FunctionT func) { + char buff[101]; + int written; + + written = func(buff, 100, "%e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000e+00"); + + written = func(buff, 100, "%E", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000E+00"); + + written = func(buff, 100, "%e", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567e+00"); + + written = func(buff, 100, "%e", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000e+00"); + + written = func(buff, 100, "%e", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000e+00"); + + written = func(buff, 100, "%e", 1e300); + ASSERT_STREQ_LEN(written, buff, "1.000000e+300"); + + written = func(buff, 100, "%e", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.234568e+18"); + + // Precision Tests + written = func(buff, 100, "%.1e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.0e+00"); + + written = func(buff, 100, "%.1e", 1.99); + ASSERT_STREQ_LEN(written, buff, "2.0e+00"); + + written = func(buff, 100, "%.1e", 9.99); + ASSERT_STREQ_LEN(written, buff, "1.0e+01"); + } + + void floatDecimalExpLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 90, "%.9e", 1000000000500000000.1L); + ASSERT_STREQ_LEN(written, buff, "1.000000001e+18"); + + written = func(buff, 90, "%.9e", 1000000000500000000.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000000e+18"); + + written = func(buff, 90, "%e", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.189731e+4932"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } + + void floatDecimalAutoSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9g", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794e+09"); + + written = func(buff, 20, "%.9G", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794E+09"); + } + + void floatDecimalAutoDoublePrec(FunctionT func) { + char buff[120]; + int written; + + written = func(buff, 100, "%g", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.23457e+18"); + + written = func(buff, 100, "%g", 9999990000000.00); + ASSERT_STREQ_LEN(written, buff, "9.99999e+12"); + + written = func(buff, 100, "%g", 9999999000000.00); + ASSERT_STREQ_LEN(written, buff, "1e+13"); + + written = func(buff, 100, "%g", 0xa.aaaaaaaaaaaaaabp-7); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 100, "%g", 0.00001); + ASSERT_STREQ_LEN(written, buff, "1e-05"); + + // Precision Tests + written = func(buff, 100, "%.0g", 0.0); + ASSERT_STREQ_LEN(written, buff, "0"); + + written = func(buff, 100, "%.2g", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.1"); + + written = func(buff, 100, "%.2g", 1.09); + ASSERT_STREQ_LEN(written, buff, "1.1"); + + written = func(buff, 100, "%.15g", 22.25); + ASSERT_STREQ_LEN(written, buff, "22.25"); + + written = func(buff, 100, "%.20g", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "1.2340000000000000814e-10"); + } + + void floatDecimalAutoLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 99, "%g", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.18973e+4932"); + + written = func(buff, 99, "%g", 0xa.aaaaaaaaaaaaaabp-7L); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 99, "%g", 9.99999999999e-100L); + ASSERT_STREQ_LEN(written, buff, "1e-99"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } +}; + +#define STRFROM_TEST(InputType, name, func) \ + using LlvmLibc##name##Test = StrfromTest; \ + TEST_F(LlvmLibc##name##Test, FloatDecimalFormat) { \ + floatDecimalFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatHexExpFormat) { floatHexExpFormat(func); } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalAutoFormat) { \ + floatDecimalAutoFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalExpFormat) { \ + floatDecimalExpFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, ImproperFormatString) { \ + improperFormatString(func); \ + } \ + TEST_F(LlvmLibc##name##Test, InsufficientBufferSize) { \ + insufficentBufsize(func); \ + } diff --git a/libc/test/src/stdlib/strfromd_test.cpp b/libc/test/src/stdlib/strfromd_test.cpp new file mode 100644 index 000000000000..55724d7e902b --- /dev/null +++ b/libc/test/src/stdlib/strfromd_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfromd --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfromd.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(double, Strfromd, LIBC_NAMESPACE::strfromd) diff --git a/libc/test/src/stdlib/strfromf_test.cpp b/libc/test/src/stdlib/strfromf_test.cpp index c5489f5f3af2..8b987fd434ac 100644 --- a/libc/test/src/stdlib/strfromf_test.cpp +++ b/libc/test/src/stdlib/strfromf_test.cpp @@ -6,102 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "StrfromTest.h" #include "src/stdlib/strfromf.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStrfromfTest, DecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 16, "%f", 1.0); - EXPECT_EQ(written, 8); - ASSERT_STREQ(buff, "1.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234567936.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 5, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234"); - - written = LIBC_NAMESPACE::strfromf(buff, 67, "%.3f", 1.0); - EXPECT_EQ(written, 5); - ASSERT_STREQ(buff, "1.000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%1f", 1234567890.0); - EXPECT_EQ(written, 3); - ASSERT_STREQ(buff, "%1f"); -} - -TEST(LlvmLibcStrfromfTest, HexExpFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0x1.26580cp+30"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%A", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0X1.26580CP+30"); -} - -TEST(LlvmLibcStrfromfTest, DecimalExpFloatFormat) { - char buff[100]; - int written; - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9e", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9E", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936E+09"); -} - -TEST(LlvmLibcStrfromfTest, AutoDecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9g", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9G", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794E+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%G", 1.0); - EXPECT_EQ(written, 1); -} - -TEST(LlvmLibcStrfromfTest, ImproperFormatString) { - - char buff[100]; - int retval; - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "A simple string with no conversions.", 1.0); - EXPECT_EQ(retval, 36); - ASSERT_STREQ(buff, "A simple string with no conversions."); - - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "%A simple string with one conversion, should overwrite.", 1.0); - EXPECT_EQ(retval, 6); - ASSERT_STREQ(buff, "0X1P+0"); - - retval = LIBC_NAMESPACE::strfromf(buff, 74, - "A simple string with one conversion in %A " - "between, writes string as it is", - 1.0); - EXPECT_EQ(retval, 73); - ASSERT_STREQ(buff, "A simple string with one conversion in %A between, " - "writes string as it is"); - - retval = LIBC_NAMESPACE::strfromf(buff, 36, - "A simple string with one conversion", 1.0); - EXPECT_EQ(retval, 35); - ASSERT_STREQ(buff, "A simple string with one conversion"); -} +STRFROM_TEST(float, StrFromf, LIBC_NAMESPACE::strfromf) diff --git a/libc/test/src/stdlib/strfroml_test.cpp b/libc/test/src/stdlib/strfroml_test.cpp new file mode 100644 index 000000000000..cf472a39a5bf --- /dev/null +++ b/libc/test/src/stdlib/strfroml_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfroml --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfroml.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(long double, Strfroml, LIBC_NAMESPACE::strfroml) -- GitLab From e64e15ee597370a9731fcba0b2b8a514f26125e7 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:39:42 -0700 Subject: [PATCH 198/933] [RISCV] Move the RISCVSchedule.td include after RISCVRegisterInfo.td. NFC Registers shouldn't depend on the scheduler, but a scheduler predicate could depend on a register. This would make it possible to move VLDSX0Pred out of the SiFive7 scheduler model to RISCVSchedule.td if another model needed it. --- llvm/lib/Target/RISCV/RISCV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 22736edc5f07..9fb84efd5b6f 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -24,8 +24,8 @@ include "RISCVSystemOperands.td" // Registers, calling conventions, instruction descriptions. //===----------------------------------------------------------------------===// -include "RISCVSchedule.td" include "RISCVRegisterInfo.td" +include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" include "GISel/RISCVRegisterBanks.td" -- GitLab From 2120f574103c487787390263b3692c4b167f6bdf Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 22 Mar 2024 11:45:51 -0700 Subject: [PATCH 199/933] Reapply [workflows] Split pr-code-format into two parts to make it more secure (#78215) (#80495) Actions triggered by pull_request_target events have access to all repository secrets, so it is unsafe to use them when executing untrusted code. The pr-code-format workflow does not execute any untrusted code, but it passes untrused input into clang-format. An attacker could use this to exploit a flaw in clang-format and potentially gain access to the repository secrets. By splitting the workflow, we can use the pull_request target which is more secure and isolate the issue write permissions in a separate job. The pull_request target also makes it easier to test changes to the code-format-helepr.py script, because the version of the script from the pull request will be used rather than the version of the script from main. Fixes #77142 --- .github/workflows/issue-write.yml | 128 +++++++++++++++++++++++++++ .github/workflows/pr-code-format.yml | 20 +++-- llvm/utils/git/code-format-helper.py | 26 ++++++ 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/issue-write.yml diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml new file mode 100644 index 000000000000..02a5f7c213e8 --- /dev/null +++ b/.github/workflows/issue-write.yml @@ -0,0 +1,128 @@ +name: Comment on an issue + +on: + workflow_run: + workflows: ["Check code formatting"] + types: + - completed + +permissions: + contents: read + +jobs: + pr-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + github-token: ${{ secrets.ISSUE_WRITE_DOWNLOAD_ARTIFACT }} + run-id: ${{ github.event.workflow_run.id }} + name: workflow-args + + - name: 'Comment on PR' + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + const comments = JSON.parse(fs.readFileSync('./comments')); + if (!comments) { + return; + } + + let runInfo = await github.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id + }); + + console.log(runInfo); + + + // Query to find the number of the pull request that triggered this job. + // The associated pull requests are based off of the branch name, so if + // you create a pull request for a branch, close it, and then create + // another pull request with the same branch, then this query will return + // two associated pull requests. This is why we have to fetch all the + // associated pull requests and then iterate through them to find the + // one that is open. + const gql_query = ` + query($repo_owner : String!, $repo_name : String!, $branch: String!) { + repository(owner: $repo_owner, name: $repo_name) { + ref (qualifiedName: $branch) { + associatedPullRequests(first: 100) { + nodes { + baseRepository { + owner { + login + } + } + number + state + } + } + } + } + } + ` + const gql_variables = { + repo_owner: runInfo.data.head_repository.owner.login, + repo_name: runInfo.data.head_repository.name, + branch: runInfo.data.head_branch + } + const gql_result = await github.graphql(gql_query, gql_variables); + console.log(gql_result); + console.log(gql_result.repository.ref.associatedPullRequests.nodes); + + var pr_number = 0; + gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => { + if (pr.baseRepository.owner.login = context.repo.owner && pr.state == 'OPEN') { + pr_number = pr.number; + } + }); + if (pr_number == 0) { + console.log("Error retrieving pull request number"); + return; + } + + await comments.forEach(function (comment) { + if (comment.id) { + // Security check: Ensure that this comment was created by + // the github-actions bot, so a malicious input won't overwrite + // a user's comment. + github.issues.getComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id + }).then((old_comment) => { + console.log(old_comment); + if (old_comment.data.user.login != "github-actions[bot]") { + console.log("Invalid comment id: " + comment.id); + return; + } + github.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + comment_id: comment.id, + body: comment.body + }); + }); + } else { + github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + body: comment.body + }); + } + }); + + - name: Dump comments file + if: always() + run: cat comments diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml index 1d1fa2483b65..2ed9b05cac12 100644 --- a/.github/workflows/pr-code-format.yml +++ b/.github/workflows/pr-code-format.yml @@ -1,12 +1,9 @@ name: "Check code formatting" on: - pull_request_target: + pull_request: branches: - main -permissions: - pull-requests: write - jobs: code_formatter: runs-on: ubuntu-latest @@ -31,12 +28,13 @@ jobs: separator: "," skip_initial_fetch: true - # We need to make sure that we aren't executing/using any code from the - # PR for security reasons as we're using pull_request_target. Checkout - # the target branch with the necessary files. + # We need to pull the script from the main branch, so that we ensure + # we get the latest version of this script. - name: Fetch code formatting utils uses: actions/checkout@v4 with: + reository: ${{ github.repository }} + ref: ${{ github.base_ref }} sparse-checkout: | llvm/utils/git/requirements_formatting.txt llvm/utils/git/code-format-helper.py @@ -77,8 +75,16 @@ jobs: # the merge base. run: | python ./code-format-tools/llvm/utils/git/code-format-helper.py \ + --write-comment-to-file \ --token ${{ secrets.GITHUB_TOKEN }} \ --issue-number $GITHUB_PR_NUMBER \ --start-rev $(git merge-base $START_REV $END_REV) \ --end-rev $END_REV \ --changed-files "$CHANGED_FILES" + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() + with: + name: workflow-args + path: | + comments diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py index 1113bf02570b..af1bb3b5aec5 100755 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -44,6 +44,7 @@ class FormatArgs: token: str = None verbose: bool = True issue_number: int = 0 + write_comment_to_file: bool = False def __init__(self, args: argparse.Namespace = None) -> None: if not args is None: @@ -53,12 +54,14 @@ class FormatArgs: self.token = args.token self.changed_files = args.changed_files self.issue_number = args.issue_number + self.write_comment_to_file = args.write_comment_to_file class FormatHelper: COMMENT_TAG = "" name: str friendly_name: str + comment: dict = None @property def comment_tag(self) -> str: @@ -119,6 +122,13 @@ View the diff from {self.name} here. comment_text = self.comment_tag + "\n\n" + comment_text existing_comment = self.find_comment(pr) + + if args.write_comment_to_file: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id + return + if existing_comment: existing_comment.edit(comment_text) elif create_new: @@ -310,6 +320,8 @@ def hook_main(): if fmt.has_tool(): if not fmt.run(args.changed_files, args): failed_fmts.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) else: print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) @@ -350,6 +362,11 @@ if __name__ == "__main__": type=str, help="Comma separated list of files that has been changed", ) + parser.add_argument( + "--write-comment-to-file", + action="store_true", + help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'", + ) args = FormatArgs(parser.parse_args()) @@ -358,9 +375,18 @@ if __name__ == "__main__": changed_files = args.changed_files.split(",") failed_formatters = [] + comments = [] for fmt in ALL_FORMATTERS: if not fmt.run(changed_files, args): failed_formatters.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) + + if len(comments): + with open("comments", "w") as f: + import json + + json.dump(comments, f) if len(failed_formatters) > 0: print(f"error: some formatters failed: {' '.join(failed_formatters)}") -- GitLab From 80fc61270d6cae8d1bfbd9211727fe1d22fc0cd5 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 11:46:06 -0700 Subject: [PATCH 200/933] Revert "[bazel] Update to 7.x (#86297)" (#86325) Reverting for https://github.com/llvm/llvm-project/pull/86297#issuecomment-2015660662 This reverts commit ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39. --- utils/bazel/.bazelrc | 5 ----- utils/bazel/.bazelversion | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index e8d055ec2322..1d7cf4a4df1b 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,11 +6,6 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### - -# Flip off to disable MODULE.bazel until we're ready. -# https://github.com/llvm/llvm-project/issues/55924 -common --enable_bzlmod=false - # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 21c8c7b46b89..5e3254243a3b 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -7.1.1 +6.1.2 -- GitLab From 50e6218132d6d9cfc2645fa19cf72d0437a62d95 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 11:42:15 -0700 Subject: [PATCH 201/933] Reland "[llvm-exegesis] Add thread IDs to subprocess memory names (#84451)" This reverts commit 1fe9c417a0bf143f9bb9f9e1fbf7b20f44196883. This relands commit 6bbe8a296ee91754d423c59c35727eaa624f7140. This was causing build failures on one of the ARMv8 builders. Still not completely sure why, but relanding it to see if the failure pops up again. If it does, the plan is to fix forward by disabling tests on ARM temporarily as llvm-exegesis does not currently use SubprocessMemory on ARM. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 9 +++--- .../llvm-exegesis/lib/SubprocessMemory.cpp | 30 +++++++++++++------ .../llvm-exegesis/lib/SubprocessMemory.h | 5 +++- .../X86/SubprocessMemoryTest.cpp | 5 +++- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 6d5f0286fb9c..498308e2edbe 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -405,6 +405,7 @@ private: if (AddMemDefError) return AddMemDefError; + long ParentTID = SubprocessMemory::getCurrentTID(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { @@ -418,7 +419,7 @@ private: // Unregister handlers, signal handling is now handled through ptrace in // the host process. sys::unregisterHandlers(); - runChildSubprocess(PipeFiles[0], Key); + runChildSubprocess(PipeFiles[0], Key, ParentTID); // The child process terminates in the above function, so we should never // get to this point. llvm_unreachable("Child process didn't exit when expected."); @@ -439,8 +440,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void runChildSubprocess(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, const BenchmarkKey &Key, + long ParentTID) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the @@ -497,7 +498,7 @@ private: Expected AuxMemFDOrError = SubprocessMemory::setupAuxiliaryMemoryInSubprocess( - Key.MemoryValues, ParentPID, CounterFileDescriptor); + Key.MemoryValues, ParentPID, ParentTID, CounterFileDescriptor); if (!AuxMemFDOrError) exit(ChildProcessExitCodeE::AuxiliaryMemorySetupFailed); diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index a49fa077257d..1fd81bd407be 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -9,11 +9,13 @@ #include "SubprocessMemory.h" #include "Error.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" #include #ifdef __linux__ #include #include +#include #include #endif @@ -22,12 +24,21 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) +long SubprocessMemory::getCurrentTID() { + // We're using the raw syscall here rather than the gettid() function provided + // by most libcs for compatibility as gettid() was only added to glibc in + // version 2.30. + return syscall(SYS_gettid); +} + Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. // This comes up particularly often when running the exegesis tests with - // llvm-lit - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ProcessID); + // llvm-lit. Additionally add the TID so that downstream consumers + // using multiple threads don't run into conflicts. + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -47,8 +58,8 @@ Error SubprocessMemory::addMemoryDefinition( pid_t ProcessPID) { SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string SharedMemoryName = "/" + std::to_string(ProcessPID) + "memdef" + - std::to_string(MemVal.Index); + std::string SharedMemoryName = + formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -82,8 +93,9 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ParentPID); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", ParentTID, ParentPID); int AuxiliaryMemoryFileDescriptor = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) @@ -97,8 +109,8 @@ Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( return make_error("Mapping auxiliary memory failed"); AuxiliaryMemoryMapping[0] = CounterFileDescriptor; for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string MemoryValueName = "/" + std::to_string(ParentPID) + "memdef" + - std::to_string(MemVal.Index); + std::string MemoryValueName = + formatv("/{0}t{1}memdef{2}", ParentPID, ParentTID, MemVal.Index); AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] = shm_open(MemoryValueName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] == -1) @@ -133,7 +145,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index e20b50cdc811..572d1085d9cf 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,6 +35,9 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; + // Gets the thread ID for the calling thread. + static long getCurrentTID(); + Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -54,7 +57,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor); ~SubprocessMemory(); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp index c07ec188a602..7c23e7b7e9c5 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #endif // __linux__ @@ -49,7 +50,9 @@ protected: std::string getSharedMemoryName(const unsigned TestNumber, const unsigned DefinitionNumber) { - return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "memdef" + + long CurrentTID = syscall(SYS_gettid); + return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "t" + + std::to_string(CurrentTID) + "memdef" + std::to_string(DefinitionNumber); } -- GitLab From 4b907414d2c0445ac1ae11811aaa4aaa2e060cf9 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Fri, 22 Mar 2024 19:59:22 +0100 Subject: [PATCH 202/933] [SystemZ] Add support for llvm.readcyclecounter The llvm.readcyclecounter intrinsic can be implemented via the STORE CLOCK FAST (STCKF) instruction. --- .../Target/SystemZ/SystemZISelLowering.cpp | 38 +++++++++++++++++++ llvm/lib/Target/SystemZ/SystemZISelLowering.h | 4 ++ llvm/lib/Target/SystemZ/SystemZInstrSystem.td | 2 +- llvm/lib/Target/SystemZ/SystemZOperators.td | 4 ++ llvm/test/CodeGen/SystemZ/readcyclecounter.ll | 27 +++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/SystemZ/readcyclecounter.ll diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index bc60d14ee700..da4bcd7f0c66 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -372,6 +372,9 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, // Handle prefetches with PFD or PFDRL. setOperationAction(ISD::PREFETCH, MVT::Other, Custom); + // Handle readcyclecounter with STCKF. + setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) { // Assume by default that all vector operations need to be expanded. for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode) @@ -6077,6 +6080,27 @@ SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op, return getCCResult(DAG, Intr); } +SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op.getOperand(0); + + // STCKF only supports a memory operand, so we have to use a temporary. + SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64); + int SPFI = cast(StackPtr.getNode())->getIndex(); + MachinePointerInfo MPI = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); + + // Use STCFK to store the TOD clock into the temporary. + SDValue StoreOps[] = {Chain, StackPtr}; + Chain = DAG.getMemIntrinsicNode( + SystemZISD::STCKF, DL, DAG.getVTList(MVT::Other), StoreOps, MVT::i64, + MPI, MaybeAlign(), MachineMemOperand::MOStore); + + // And read it back from there. + return DAG.getLoad(MVT::i64, DL, Chain, StackPtr, MPI); +} + SDValue SystemZTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { switch (Op.getOpcode()) { @@ -6199,6 +6223,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op, return lowerIS_FPCLASS(Op, DAG); case ISD::GET_ROUNDING: return lowerGET_ROUNDING(Op, DAG); + case ISD::READCYCLECOUNTER: + return lowerREADCYCLECOUNTER(Op, DAG); default: llvm_unreachable("Unexpected node to lower"); } @@ -6425,6 +6451,7 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(STRV); OPCODE(VLER); OPCODE(VSTER); + OPCODE(STCKF); OPCODE(PREFETCH); OPCODE(ADA_ENTRY); } @@ -6985,6 +7012,17 @@ SDValue SystemZTargetLowering::combineSTORE( } } + // Combine STORE (READCYCLECOUNTER) into STCKF. + if (!SN->isTruncatingStore() && + Op1.getOpcode() == ISD::READCYCLECOUNTER && + Op1.hasOneUse() && + N->getOperand(0).reachesChainWithoutSideEffects(SDValue(Op1.getNode(), 1))) { + SDValue Ops[] = { Op1.getOperand(0), N->getOperand(2) }; + return DAG.getMemIntrinsicNode(SystemZISD::STCKF, SDLoc(N), + DAG.getVTList(MVT::Other), + Ops, MemVT, SN->getMemOperand()); + } + // Transform a store of an i128 moved from GPRs into two separate stores. if (MemVT == MVT::i128 && SN->isSimple() && ISD::isNormalStore(SN)) { SDValue LoPart, HiPart; diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 406a13b9281c..4943c5cb703c 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -378,6 +378,9 @@ enum NodeType : unsigned { // Element swapping load/store. Same operands as regular load/store. VLER, VSTER, + // Use STORE CLOCK FAST to store current TOD clock value. + STCKF, + // Prefetch from the second operand using the 4-bit control code in // the first operand. The code is 1 for a load prefetch and 2 for // a store prefetch. @@ -717,6 +720,7 @@ private: SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const; SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const; SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; bool canTreatAsByteVector(EVT VT) const; SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td index e26417ddb72d..497844bec85d 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td @@ -339,7 +339,7 @@ let hasSideEffects = 1 in // Store clock (fast / extended). let hasSideEffects = 1, Defs = [CC] in { def STCK : StoreInherentS<"stck", 0xB205, null_frag, 8>; - def STCKF : StoreInherentS<"stckf", 0xB27C, null_frag, 8>; + def STCKF : StoreInherentS<"stckf", 0xB27C, z_stckf, 8>; def STCKE : StoreInherentS<"stcke", 0xB278, null_frag, 16>; } diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td index 6c4e33a6aa7f..1611436b01b7 100644 --- a/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -132,6 +132,8 @@ def SDT_ZIPM : SDTypeProfile<1, 1, def SDT_ZPrefetch : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>]>; +def SDT_ZStoreInherent : SDTypeProfile<0, 1, + [SDTCisPtrTy<0>]>; def SDT_ZTBegin : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, @@ -307,6 +309,8 @@ def z_loadeswap : SDNode<"SystemZISD::VLER", SDTLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def z_storeeswap : SDNode<"SystemZISD::VSTER", SDTStore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def z_stckf : SDNode<"SystemZISD::STCKF", SDT_ZStoreInherent, + [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; def z_tdc : SDNode<"SystemZISD::TDC", SDT_ZTest>; diff --git a/llvm/test/CodeGen/SystemZ/readcyclecounter.ll b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll new file mode 100644 index 000000000000..34b6d34143f8 --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll @@ -0,0 +1,27 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=s390x-ibm-linux | FileCheck %s + +; Verify that we correctly lower ISD::READCYCLECOUNTER. + +define i64 @test_builtin_readcyclecounter1() { +; CHECK-LABEL: test_builtin_readcyclecounter1: +; CHECK: # %bb.0: +; CHECK-NEXT: aghi %r15, -168 +; CHECK-NEXT: .cfi_def_cfa_offset 328 +; CHECK-NEXT: stckf 160(%r15) +; CHECK-NEXT: lg %r2, 160(%r15) +; CHECK-NEXT: aghi %r15, 168 +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + ret i64 %1 +} + +define void @test_builtin_readcyclecounter2(ptr %ptr) { +; CHECK-LABEL: test_builtin_readcyclecounter2: +; CHECK: # %bb.0: +; CHECK-NEXT: stckf 0(%r2) +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + store i64 %1, ptr %ptr + ret void +} -- GitLab From ee9fa38554f8427f5361708b28551e70ff0e1256 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Fri, 22 Mar 2024 15:20:00 -0400 Subject: [PATCH 203/933] [libc] fix missing macro dependency in bazel (#86298) ![image](https://github.com/llvm/llvm-project/assets/20108837/94cb5718-3526-4bae-8a79-f5b1d19b352d) --- .../llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel index 803010e8b3ad..5f59d70ecc16 100644 --- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel @@ -46,6 +46,7 @@ libc_support_library( "//libc:__support_cpp_type_traits", "//libc:__support_fputil_fp_bits", "//libc:__support_fputil_fpbits_str", + "//libc:llvm_libc_macros_math_macros", "//libc/test/UnitTest:LibcUnitTest", "//libc/test/UnitTest:fp_test_helpers", "//libc/utils/MPFRWrapper:mpfr_impl", -- GitLab From d394f3a162b871668d0c8e8bf6a94922fa8698ae Mon Sep 17 00:00:00 2001 From: Xing Xue Date: Fri, 22 Mar 2024 15:25:08 -0400 Subject: [PATCH 204/933] [OpenMP][AIX] Affinity implementation for AIX (#84984) This patch implements `affinity` for AIX, which is quite different from platforms such as Linux. - Setting CPU affinity through masks and related functions are not supported. System call `bindprocessor()` is used to bind a thread to one CPU per call. - There are no system routines to get the affinity info of a thread. The implementation of `get_system_affinity()` for AIX gets the mask of all available CPUs, to be used as the full mask only. - Topology is not available from the file system. It is obtained through system SRAD (Scheduler Resource Allocation Domain). This patch has run through the libomp LIT tests successfully with `affinity` enabled. --- openmp/runtime/src/kmp.h | 5 +- openmp/runtime/src/kmp_affinity.cpp | 130 ++++++++++++++++++++++++++-- openmp/runtime/src/kmp_affinity.h | 74 +++++++++++++++- openmp/runtime/src/kmp_os.h | 2 +- openmp/runtime/src/z_Linux_util.cpp | 39 +++++++-- openmp/runtime/test/lit.cfg | 2 +- 6 files changed, 233 insertions(+), 19 deletions(-) diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index 885d6636abe4..18ccf10fe17d 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -819,6 +819,7 @@ private: typedef KMPAffinity::Mask kmp_affin_mask_t; extern KMPAffinity *__kmp_affinity_dispatch; +#ifndef KMP_OS_AIX class kmp_affinity_raii_t { kmp_affin_mask_t *mask; bool restored; @@ -843,6 +844,7 @@ public: } ~kmp_affinity_raii_t() { restore(); } }; +#endif // !KMP_OS_AIX // Declare local char buffers with this size for printing debug and info // messages, using __kmp_affinity_print_mask(). @@ -3910,7 +3912,8 @@ extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size); #if KMP_WEIGHTED_ITERATIONS_SUPPORTED extern int __kmp_get_first_osid_with_ecore(void); #endif -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX extern int kmp_set_thread_affinity_mask_initial(void); #endif static inline void __kmp_assign_root_init_mask() { diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index 048bd174fc95..b574dbbaf54f 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -2910,12 +2910,17 @@ static inline const char *__kmp_cpuinfo_get_envvar() { } // Parse /proc/cpuinfo (or an alternate file in the same format) to obtain the -// affinity map. +// affinity map. On AIX, the map is obtained through system SRAD (Scheduler +// Resource Allocation Domain). static bool __kmp_affinity_create_cpuinfo_map(int *line, kmp_i18n_id_t *const msg_id) { + *msg_id = kmp_i18n_null; + +#if KMP_OS_AIX + unsigned num_records = __kmp_xproc; +#else const char *filename = __kmp_cpuinfo_get_filename(); const char *envvar = __kmp_cpuinfo_get_envvar(); - *msg_id = kmp_i18n_null; if (__kmp_affinity.flags.verbose) { KMP_INFORM(AffParseFilename, "KMP_AFFINITY", filename); @@ -2974,6 +2979,7 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, *msg_id = kmp_i18n_str_CantRewindCpuinfo; return false; } +#endif // KMP_OS_AIX // Allocate the array of records to store the proc info in. The dummy // element at the end makes the logic in filling them out easier to code. @@ -3003,6 +3009,99 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, INIT_PROC_INFO(threadInfo[i]); } +#if KMP_OS_AIX + int smt_threads; + lpar_info_format1_t cpuinfo; + unsigned num_avail = __kmp_xproc; + + if (__kmp_affinity.flags.verbose) + KMP_INFORM(AffParseFilename, "KMP_AFFINITY", "system info for topology"); + + // Get the number of SMT threads per core. + int retval = + lpar_get_info(LPAR_INFO_FORMAT1, &cpuinfo, sizeof(lpar_info_format1_t)); + if (!retval) + smt_threads = cpuinfo.smt_threads; + else { + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Allocate a resource set containing available system resourses. + rsethandle_t sys_rset = rs_alloc(RS_SYSTEM); + if (sys_rset == NULL) { + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + // Allocate a resource set for the SRAD info. + rsethandle_t srad = rs_alloc(RS_EMPTY); + if (srad == NULL) { + rs_free(sys_rset); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Get the SRAD system detail level. + int sradsdl = rs_getinfo(NULL, R_SRADSDL, 0); + if (sradsdl < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + // Get the number of RADs at that SRAD SDL. + int num_rads = rs_numrads(sys_rset, sradsdl, 0); + if (num_rads < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Get the maximum number of procs that may be contained in a resource set. + int max_procs = rs_getinfo(NULL, R_MAXPROCS, 0); + if (max_procs < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + int cur_rad = 0; + int num_set = 0; + for (int srad_idx = 0; cur_rad < num_rads && srad_idx < VMI_MAXRADS; + ++srad_idx) { + // Check if the SRAD is available in the RSET. + if (rs_getrad(sys_rset, srad, sradsdl, srad_idx, 0) < 0) + continue; + + for (int cpu = 0; cpu < max_procs; cpu++) { + // Set the info for the cpu if it is in the SRAD. + if (rs_op(RS_TESTRESOURCE, srad, NULL, R_PROCS, cpu)) { + threadInfo[cpu][osIdIndex] = cpu; + threadInfo[cpu][pkgIdIndex] = cur_rad; + threadInfo[cpu][coreIdIndex] = cpu / smt_threads; + ++num_set; + if (num_set >= num_avail) { + // Done if all available CPUs have been set. + break; + } + } + } + ++cur_rad; + } + rs_free(sys_rset); + rs_free(srad); + + // The topology is already sorted. + +#else // !KMP_OS_AIX unsigned num_avail = 0; *line = 0; #if KMP_ARCH_S390X @@ -3250,6 +3349,8 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, qsort(threadInfo, num_avail, sizeof(*threadInfo), __kmp_affinity_cmp_ProcCpuInfo_phys_id); +#endif // KMP_OS_AIX + // The table is now sorted by pkgId / coreId / threadId, but we really don't // know the radix of any of the fields. pkgId's may be sparsely assigned among // the chips on a system. Although coreId's are usually assigned @@ -4445,7 +4546,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { } #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ -#if KMP_OS_LINUX +#if KMP_OS_LINUX || KMP_OS_AIX if (!success) { int line = 0; success = __kmp_affinity_create_cpuinfo_map(&line, &msg_id); @@ -4841,7 +4942,12 @@ void __kmp_affinity_uninitialize(void) { } if (__kmp_affin_origMask != NULL) { if (KMP_AFFINITY_CAPABLE()) { +#if KMP_OS_AIX + // Uninitialize by unbinding the thread. + bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY); +#else __kmp_set_system_affinity(__kmp_affin_origMask, FALSE); +#endif } KMP_CPU_FREE(__kmp_affin_origMask); __kmp_affin_origMask = NULL; @@ -5015,7 +5121,10 @@ void __kmp_affinity_bind_init_mask(int gtid) { __kmp_set_system_affinity(th->th.th_affin_mask, FALSE); } else #endif +#ifndef KMP_OS_AIX + // Do not set the full mask as the init mask on AIX. __kmp_set_system_affinity(th->th.th_affin_mask, TRUE); +#endif } void __kmp_affinity_bind_place(int gtid) { @@ -5128,7 +5237,7 @@ int __kmp_aux_set_affinity(void **mask) { int __kmp_aux_get_affinity(void **mask) { int gtid; int retval; -#if KMP_OS_WINDOWS || KMP_DEBUG +#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG kmp_info_t *th; #endif if (!KMP_AFFINITY_CAPABLE()) { @@ -5136,7 +5245,7 @@ int __kmp_aux_get_affinity(void **mask) { } gtid = __kmp_entry_gtid(); -#if KMP_OS_WINDOWS || KMP_DEBUG +#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG th = __kmp_threads[gtid]; #else (void)gtid; // unused variable @@ -5159,7 +5268,7 @@ int __kmp_aux_get_affinity(void **mask) { } } -#if !KMP_OS_WINDOWS +#if !KMP_OS_WINDOWS && !KMP_OS_AIX retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE); KA_TRACE( @@ -5179,7 +5288,7 @@ int __kmp_aux_get_affinity(void **mask) { KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask); return 0; -#endif /* KMP_OS_WINDOWS */ +#endif /* !KMP_OS_WINDOWS && !KMP_OS_AIX */ } int __kmp_aux_get_affinity_max_proc() { @@ -5561,7 +5670,8 @@ void __kmp_balanced_affinity(kmp_info_t *th, int nthreads) { } } -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX // We don't need this entry for Windows because // there is GetProcessAffinityMask() api // @@ -5596,7 +5706,11 @@ extern "C" "set full mask for thread %d\n", gtid)); KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL); +#if KMP_OS_AIX + return bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY); +#else return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE); +#endif } #endif diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h index 1c7db2f59943..7efc090f8863 100644 --- a/openmp/runtime/src/kmp_affinity.h +++ b/openmp/runtime/src/kmp_affinity.h @@ -191,7 +191,8 @@ public: }; #endif /* KMP_USE_HWLOC */ -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX #if KMP_OS_LINUX /* On some of the older OS's that we build on, these constants aren't present in #included from . They must be the same on @@ -317,6 +318,10 @@ public: #elif KMP_OS_NETBSD #include #include +#elif KMP_OS_AIX +#include +#include +#define VMI_MAXRADS 64 // Maximum number of RADs allowed by AIX. #endif class KMPNativeAffinity : public KMPAffinity { class Mask : public KMPAffinity::Mask { @@ -404,6 +409,70 @@ class KMPNativeAffinity : public KMPAffinity { ++retval; return retval; } +#if KMP_OS_AIX + // On AIX, we don't have a way to get CPU(s) a thread is bound to. + // This routine is only used to get the full mask. + int get_system_affinity(bool abort_on_error) override { + KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), + "Illegal get affinity operation when not capable"); + + (void)abort_on_error; + + // Set the mask with all CPUs that are available. + for (int i = 0; i < __kmp_xproc; ++i) + KMP_CPU_SET(i, this); + return 0; + } + int set_system_affinity(bool abort_on_error) const override { + KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), + + "Illegal set affinity operation when not capable"); + + int location; + int gtid = __kmp_entry_gtid(); + int tid = thread_self(); + + // Unbind the thread if it was bound to any processors before so that + // we can bind the thread to CPUs specified by the mask not others. + int retval = bindprocessor(BINDTHREAD, tid, PROCESSOR_CLASS_ANY); + + // On AIX, we can only bind to one instead of a set of CPUs with the + // bindprocessor() system call. + KMP_CPU_SET_ITERATE(location, this) { + if (KMP_CPU_ISSET(location, this)) { + retval = bindprocessor(BINDTHREAD, tid, location); + if (retval == -1 && errno == 1) { + rsid_t rsid; + rsethandle_t rsh; + // Put something in rsh to prevent compiler warning + // about uninitalized use + rsh = rs_alloc(RS_EMPTY); + rsid.at_pid = getpid(); + if (RS_DEFAULT_RSET != ra_getrset(R_PROCESS, rsid, 0, rsh)) { + retval = ra_detachrset(R_PROCESS, rsid, 0); + retval = bindprocessor(BINDTHREAD, tid, location); + } + } + if (retval == 0) { + KA_TRACE(10, ("__kmp_set_system_affinity: Done binding " + "T#%d to cpu=%d.\n", + gtid, location)); + continue; + } + int error = errno; + if (abort_on_error) { + __kmp_fatal(KMP_MSG(FunctionError, "bindprocessor()"), + KMP_ERR(error), __kmp_msg_null); + KA_TRACE(10, ("__kmp_set_system_affinity: Error binding " + "T#%d to cpu=%d, errno=%d.\n", + gtid, location, error)); + return error; + } + } + } + return 0; + } +#else // !KMP_OS_AIX int get_system_affinity(bool abort_on_error) override { KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), "Illegal get affinity operation when not capable"); @@ -446,6 +515,7 @@ class KMPNativeAffinity : public KMPAffinity { } return error; } +#endif // KMP_OS_AIX }; void determine_capable(const char *env_var) override { __kmp_affinity_determine_capable(env_var); @@ -475,7 +545,7 @@ class KMPNativeAffinity : public KMPAffinity { api_type get_api_type() const override { return NATIVE_OS; } }; #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY \ - */ + || KMP_OS_AIX */ #if KMP_OS_WINDOWS class KMPNativeAffinity : public KMPAffinity { diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h index 63da9e5fa15d..a628070c882a 100644 --- a/openmp/runtime/src/kmp_os.h +++ b/openmp/runtime/src/kmp_os.h @@ -76,7 +76,7 @@ #endif #if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_DRAGONFLY) && \ + KMP_OS_DRAGONFLY || KMP_OS_AIX) && \ !KMP_OS_WASI #define KMP_AFFINITY_SUPPORTED 1 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64 diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp index d751a417331c..29db9d008a49 100644 --- a/openmp/runtime/src/z_Linux_util.cpp +++ b/openmp/runtime/src/z_Linux_util.cpp @@ -125,7 +125,8 @@ static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) { } #endif -#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY) && \ +#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX) && \ KMP_AFFINITY_SUPPORTED) /* Affinity support */ @@ -142,6 +143,29 @@ void __kmp_affinity_bind_thread(int which) { KMP_CPU_FREE_FROM_STACK(mask); } +#if KMP_OS_AIX +void __kmp_affinity_determine_capable(const char *env_var) { + // All versions of AIX support bindprocessor(). + + size_t mask_size = __kmp_xproc / CHAR_BIT; + // Round up to byte boundary. + if (__kmp_xproc % CHAR_BIT) + ++mask_size; + + // Round up to the mask_size_type boundary. + if (mask_size % sizeof(__kmp_affin_mask_size)) + mask_size += sizeof(__kmp_affin_mask_size) - + mask_size % sizeof(__kmp_affin_mask_size); + KMP_AFFINITY_ENABLE(mask_size); + KA_TRACE(10, + ("__kmp_affinity_determine_capable: " + "AIX OS affinity interface bindprocessor functional (mask size = " + "%" KMP_SIZE_T_SPEC ").\n", + __kmp_affin_mask_size)); +} + +#else // !KMP_OS_AIX + /* Determine if we can access affinity functionality on this version of * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set * __kmp_affin_mask_size to the appropriate value (0 means not capable). */ @@ -271,8 +295,9 @@ void __kmp_affinity_determine_capable(const char *env_var) { KMP_WARNING(AffCantGetMaskSize, env_var); } } - -#endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED +#endif // KMP_OS_AIX +#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ + KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED #if KMP_USE_FUTEX @@ -501,7 +526,7 @@ static void *__kmp_launch_worker(void *thr) { #endif /* KMP_BLOCK_SIGNALS */ void *exit_val; #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS + KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX void *volatile padding = 0; #endif int gtid; @@ -550,7 +575,7 @@ static void *__kmp_launch_worker(void *thr) { #endif /* KMP_BLOCK_SIGNALS */ #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS + KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX if (__kmp_stkoffset > 0 && gtid > 0) { padding = KMP_ALLOCA(gtid * __kmp_stkoffset); (void)padding; @@ -1268,7 +1293,8 @@ static void __kmp_atfork_child(void) { ++__kmp_fork_count; #if KMP_AFFINITY_SUPPORTED -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX // reset the affinity in the child to the initial thread // affinity in the parent kmp_set_thread_affinity_mask_initial(); @@ -2325,6 +2351,7 @@ int __kmp_is_address_mapped(void *addr) { found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE); #elif KMP_OS_AIX + (void)rc; // FIXME(AIX): Implement this found = 1; diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg index a3456063c10f..e27e52bb4289 100644 --- a/openmp/runtime/test/lit.cfg +++ b/openmp/runtime/test/lit.cfg @@ -129,7 +129,7 @@ if config.operating_system == 'NetBSD': if config.operating_system == 'Darwin': config.available_features.add("darwin") -if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly']: +if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly', 'AIX']: config.available_features.add('affinity') if config.operating_system in ['Linux']: -- GitLab From dfe4ca9b7f4a422500d78280dc5eefd1979939e6 Mon Sep 17 00:00:00 2001 From: Paul Kirth Date: Fri, 22 Mar 2024 12:27:41 -0700 Subject: [PATCH 205/933] [RISCV][lld] Set the type of TLSDESC relocation's referenced local symbol to STT_NOTYPE When adding fixups for RISCV_TLSDESC_ADD_LO and RISCV_TLSDESC_LOAD_LO, the local label added for RISCV TLSDESC relocations have STT_TLS set, which is incorrect. Instead, these labels should have `STT_NOTYPE`. This patch stops adding such fixups and avoid setting the STT_TLS on these symbols. Failing to do so can cause LLD to emit an error `has an STT_TLS symbol but doesn't have an SHF_TLS section`. We additionally, adjust how LLD services these relocations to avoid errors with incompatible relocation and symbol types. Reviewers: topperc, MaskRay Reviewed By: MaskRay Pull Request: https://github.com/llvm/llvm-project/pull/85817 --- lld/ELF/Relocations.cpp | 5 +++- lld/test/ELF/riscv-tlsdesc-relax.s | 8 ++++++ lld/test/ELF/riscv-tlsdesc.s | 27 +++++++++++-------- .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp | 2 -- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 619fbaf5dc54..92a1b9baaca3 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1480,7 +1480,10 @@ template void RelocationScanner::scanOne(RelTy *&i) { // Process TLS relocations, including TLS optimizations. Note that // R_TPREL and R_TPREL_NEG relocations are resolved in processAux. - if (sym.isTls()) { + // + // Some RISCV TLSDESC relocations reference a local NOTYPE symbol, + // but we need to process them in handleTlsRelocation. + if (sym.isTls() || oneof(expr)) { if (unsigned processed = handleTlsRelocation(type, sym, *sec, offset, addend, expr)) { i += processed - 1; diff --git a/lld/test/ELF/riscv-tlsdesc-relax.s b/lld/test/ELF/riscv-tlsdesc-relax.s index fb24317e6535..5718d4175be1 100644 --- a/lld/test/ELF/riscv-tlsdesc-relax.s +++ b/lld/test/ELF/riscv-tlsdesc-relax.s @@ -33,12 +33,14 @@ # GD64-NEXT: c.add a0, tp # GD64-NEXT: jal {{.*}} ## &.got[c]-. = 0x20c0+8 - 0x1020 = 0x10a8 +# GD64-LABEL: <.Ltlsdesc_hi1>: # GD64-NEXT: 1020: auipc a4, 0x1 # GD64-NEXT: ld a5, 0xa8(a4) # GD64-NEXT: addi a0, a4, 0xa8 # GD64-NEXT: jalr t0, 0x0(a5) # GD64-NEXT: c.add a0, tp ## &.got[c]-. = 0x20c0+8 - 0x1032 = 0x1096 +# GD64-LABEL: <.Ltlsdesc_hi2>: # GD64-NEXT: 1032: auipc a6, 0x1 # GD64-NEXT: ld a7, 0x96(a6) # GD64-NEXT: addi a0, a6, 0x96 @@ -64,6 +66,7 @@ # LE64-NEXT: jal {{.*}} # LE64-NEXT: R_RISCV_JAL foo # LE64-NEXT: R_RISCV_RELAX *ABS* +# LE64-LABEL: <.Ltlsdesc_hi1>: # LE64-NEXT: addi a0, zero, 0x7ff # LE64-NEXT: R_RISCV_TLSDESC_HI20 b # LE64-NEXT: R_RISCV_RELAX *ABS* @@ -71,6 +74,7 @@ # LE64-NEXT: R_RISCV_TLSDESC_ADD_LO12 .Ltlsdesc_hi1 # LE64-NEXT: R_RISCV_TLSDESC_CALL .Ltlsdesc_hi1 # LE64-NEXT: c.add a0, tp +# LE64-LABEL: <.Ltlsdesc_hi2>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: R_RISCV_TLSDESC_HI20 b # LE64-NEXT: addi zero, zero, 0x0 @@ -93,9 +97,11 @@ # LE64A-NEXT: addi a0, a0, -0x479 # LE64A-NEXT: c.add a0, tp # LE64A-NEXT: jal {{.*}} +# LE64A-LABEL: <.Ltlsdesc_hi1>: # LE64A-NEXT: lui a0, 0x2 # LE64A-NEXT: addi a0, a0, -0x479 # LE64A-NEXT: c.add a0, tp +# LE64A-LABEL: <.Ltlsdesc_hi2>: # LE64A-NEXT: addi zero, zero, 0x0 # LE64A-NEXT: addi zero, zero, 0x0 # LE64A-NEXT: lui a0, 0x2 @@ -115,10 +121,12 @@ # IE64-NEXT: c.add a0, tp # IE64-NEXT: jal {{.*}} ## &.got[c]-. = 0x120e0+8 - 0x11018 = 0x10d0 +# IE64-LABEL: <.Ltlsdesc_hi1>: # IE64-NEXT: 11018: auipc a0, 0x1 # IE64-NEXT: ld a0, 0xd0(a0) # IE64-NEXT: c.add a0, tp ## &.got[c]-. = 0x120e0+8 - 0x1102a = 0x10be +# IE64-LABEL: <.Ltlsdesc_hi2>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: 1102a: auipc a0, 0x1 diff --git a/lld/test/ELF/riscv-tlsdesc.s b/lld/test/ELF/riscv-tlsdesc.s index c583e15cf30c..935ecbddfbff 100644 --- a/lld/test/ELF/riscv-tlsdesc.s +++ b/lld/test/ELF/riscv-tlsdesc.s @@ -29,11 +29,13 @@ # RUN: ld.lld -e 0 -z now a.32.o c.32.so -o a.32.ie # RUN: llvm-objdump --no-show-raw-insn -M no-aliases -h -d a.32.ie | FileCheck %s --check-prefix=IE32 -# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o -# RUN: not ld.lld -shared -soname=d.64.so -o d.64.so d.64.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL +## Prior to https://github.com/llvm/llvm-project/pull/85817 the local TLSDESC +## labels would be marked STT_TLS, resulting in an error "has an STT_TLS symbol but doesn't have an SHF_TLS section" +# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o +# RUN: ld.lld -shared -soname=d.64.so -o d.64.so d.64.o --fatal-warnings # RUN: llvm-mc -triple=riscv32 -filetype=obj d.s -o d.32.o --defsym ELF32=1 -# RUN: not ld.lld -shared -soname=d.32.so -o d.32.so d.32.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL +# RUN: ld.lld -shared -soname=d.32.so -o d.32.so d.32.o --fatal-warnings # GD64-RELA: .rela.dyn { # GD64-RELA-NEXT: 0x2408 R_RISCV_TLSDESC - 0x7FF @@ -74,14 +76,14 @@ # GD64-NEXT: add a0, a0, tp ## &.got[b]-. = 0x23e0+40 - 0x12f4 = 0x1114 -# GD64-NEXT: 12f4: auipc a2, 0x1 +# GD64: 12f4: auipc a2, 0x1 # GD64-NEXT: ld a3, 0x114(a2) # GD64-NEXT: addi a0, a2, 0x114 # GD64-NEXT: jalr t0, 0x0(a3) # GD64-NEXT: add a0, a0, tp ## &.got[c]-. = 0x23e0+24 - 0x1308 = 0x10f0 -# GD64-NEXT: 1308: auipc a4, 0x1 +# GD64: 1308: auipc a4, 0x1 # GD64-NEXT: ld a5, 0xf0(a4) # GD64-NEXT: addi a0, a4, 0xf0 # GD64-NEXT: jalr t0, 0x0(a5) @@ -89,7 +91,7 @@ # NOREL: no relocations -# LE64-LABEL: <.text>: +# LE64-LABEL: <.Ltlsdesc_hi0>: ## st_value(a) = 8 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 @@ -97,12 +99,14 @@ # LE64-NEXT: addi a0, zero, 0x8 # LE64-NEXT: add a0, a0, tp ## st_value(b) = 2047 +# LE64-LABEL: <.Ltlsdesc_hi1>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi a0, zero, 0x7ff # LE64-NEXT: add a0, a0, tp ## st_value(c) = 2048 +# LE64-LABEL: <.Ltlsdesc_hi2>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: lui a0, 0x1 @@ -116,18 +120,20 @@ # IE64: .got 00000010 00000000000123a8 ## a and b are optimized to use LE. c is optimized to IE. -# IE64-LABEL: <.text>: +# IE64-LABEL: <.Ltlsdesc_hi0>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi a0, zero, 0x8 # IE64-NEXT: add a0, a0, tp +# IE64-LABEL: <.Ltlsdesc_hi1>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi a0, zero, 0x7ff # IE64-NEXT: add a0, a0, tp ## &.got[c]-. = 0x123a8+8 - 0x112b8 = 0x10f8 +# IE64-LABEL: <.Ltlsdesc_hi2>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: 112b8: auipc a0, 0x1 @@ -136,7 +142,7 @@ # IE32: .got 00000008 00012248 -# IE32-LABEL: <.text>: +# IE32-LABEL: <.Ltlsdesc_hi0>: ## st_value(a) = 8 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 @@ -144,21 +150,20 @@ # IE32-NEXT: addi a0, zero, 0x8 # IE32-NEXT: add a0, a0, tp ## st_value(b) = 2047 +# IE32-LABEL: <.Ltlsdesc_hi1>: # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi a0, zero, 0x7ff # IE32-NEXT: add a0, a0, tp ## &.got[c]-. = 0x12248+4 - 0x111cc = 0x1080 +# IE32-LABEL: <.Ltlsdesc_hi2>: # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: 111cc: auipc a0, 0x1 # IE32-NEXT: lw a0, 0x80(a0) # IE32-NEXT: add a0, a0, tp -## FIXME This should not pass, but the code MC layer needs a fix to prevent this. -# BADTLSLABEL: error: d.{{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section - #--- a.s .macro load dst, src .ifdef ELF32 diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp index 254a9a4bc0ef..b8e0f3a867f4 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp @@ -207,8 +207,6 @@ void RISCVMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const { case VK_RISCV_TLS_GOT_HI: case VK_RISCV_TLS_GD_HI: case VK_RISCV_TLSDESC_HI: - case VK_RISCV_TLSDESC_ADD_LO: - case VK_RISCV_TLSDESC_LOAD_LO: break; } -- GitLab From e9639e9c0636d9e2b9591c2cdac5cac75e363e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Fri, 22 Mar 2024 12:56:45 -0700 Subject: [PATCH 206/933] [flang][NFC] Extract FIROpConversion to its own files (#86213) This PR extracts `FIROpConversion` and `FIROpAndTypeConversion` templated base patterns to a header file. All the functions from FIROpConversion that do not require the template argument are moved to a base class named `ConvertFIRToLLVMPattern`. This move is done so the `FIROpConversion` pattern and all its utility functions can be reused outside of the codegen pass. For the most part the code is only moved to the new files and not modified. The only update is that addition of the PatternBenefit argument with a default value to the constructor so it can be forwarded to the `ConversionPattern` ctor. This split is done in a similar way for the `ConvertOpToLLVMPattern` base pattern that is based on the `ConvertToLLVMPattern` base class in `mlir/include/mlir/Conversion/LLVMCommon/Pattern.h`. --- .../flang/Optimizer/CodeGen/FIROpPatterns.h | 248 +++++++++ .../flang/Optimizer/CodeGen/TypeConverter.h | 4 +- flang/lib/Optimizer/CodeGen/CMakeLists.txt | 1 + flang/lib/Optimizer/CodeGen/CodeGen.cpp | 486 +++--------------- flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp | 315 ++++++++++++ 5 files changed, 636 insertions(+), 418 deletions(-) create mode 100644 flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h create mode 100644 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h new file mode 100644 index 000000000000..06a44f188565 --- /dev/null +++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h @@ -0,0 +1,248 @@ +//===-- FIROpPatterns.h -- FIR operation conversion patterns ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H +#define FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H + +#include "flang/Optimizer/CodeGen/TypeConverter.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +namespace fir { + +struct FIRToLLVMPassOptions; + +static constexpr unsigned defaultAddressSpace = 0u; + +class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern { +public: + ConvertFIRToLLVMPattern(llvm::StringRef rootOpName, + mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1); + +protected: + mlir::Type convertType(mlir::Type ty) const { + return lowerTy().convertType(ty); + } + + // Convert FIR type to LLVM without turning fir.box into memory + // reference. + mlir::Type convertObjectType(mlir::Type firType) const; + + mlir::LLVM::ConstantOp + genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const; + + mlir::LLVM::ConstantOp + genConstantOffset(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + int offset) const; + + /// Perform an extension or truncation as needed on an integer value. Lowering + /// to the specific target may involve some sign-extending or truncation of + /// values, particularly to fit them from abstract box types to the + /// appropriate reified structures. + mlir::Value integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const; + struct TypePair { + mlir::Type fir; + mlir::Type llvm; + }; + + TypePair getBoxTypePair(mlir::Type firBoxTy) const; + + /// Construct code sequence to extract the specific value from a `fir.box`. + mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, + int boxValue) const; + + /// Method to construct code sequence to get the triple for dimension `dim` + /// from a box. + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const; + + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, int dim, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::Value dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + int dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value getStrideFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const; + + /// Read base address from a fir.box. Returned address has type ty. + mlir::Value + getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Get the element type given an LLVM type that is of the form + // (array|struct|vector)+ and the provided indexes. + mlir::Type getBoxEleTy(mlir::Type type, + llvm::ArrayRef indexes) const; + + // Return LLVM type of the object described by a fir.box of \p boxType. + mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const; + + /// Read the address of the type descriptor from a box. + mlir::Value + loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Load the attribute from the \p box and perform a check against \p maskValue + // The final comparison is implemented as `(attribute & maskValue) != 0`. + mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, + unsigned maskValue) const; + + template + mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter, + mlir::Value base, ARGS... args) const { + llvm::SmallVector cv = {args...}; + auto llvmPtrTy = + mlir::LLVM::LLVMPointerType::get(ty.getContext(), /*addressSpace=*/0); + return rewriter.create(loc, llvmPtrTy, ty, base, cv); + } + + // Find the Block in which the alloca should be inserted. + // The order to recursively find the proper block: + // 1. An OpenMP Op that will be outlined. + // 2. A LLVMFuncOp + // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp + mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) const; + + // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the + // allocation address space provided for the architecture in the DataLayout + // specification. If the address space is different from the devices + // program address space we perform a cast. In the case of most architectures + // the program and allocation address space will be the default of 0 and no + // cast will be emitted. + mlir::Value + genAllocaAndAddrCastWithType(mlir::Location loc, mlir::Type llvmObjectTy, + unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const; + + const fir::LLVMTypeConverter &lowerTy() const { + return *static_cast( + this->getTypeConverter()); + } + + void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, + mlir::Type baseFIRType, mlir::Type accessFIRType, + mlir::LLVM::GEPOp gep) const { + lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); + } + + unsigned + getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + unsigned + getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + const fir::FIRToLLVMPassOptions &options; + + using ConvertToLLVMPattern::match; + using ConvertToLLVMPattern::matchAndRewrite; +}; + +template +class FIROpConversion : public ConvertFIRToLLVMPattern { +public: + using OpAdaptor = typename SourceOp::Adaptor; + + explicit FIROpConversion(const LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1) + : ConvertFIRToLLVMPattern(SourceOp::getOperationName(), + &typeConverter.getContext(), typeConverter, + options, benefit) {} + + /// Wrappers around the RewritePattern methods that pass the derived op type. + void rewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + rewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), rewriter); + } + mlir::LogicalResult match(mlir::Operation *op) const final { + return match(mlir::cast(op)); + } + mlir::LogicalResult + matchAndRewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + return matchAndRewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), + rewriter); + } + + /// Rewrite and Match methods that operate on the SourceOp type. These must be + /// overridden by the derived pattern class. + virtual mlir::LogicalResult match(SourceOp op) const { + llvm_unreachable("must override match or matchAndRewrite"); + } + virtual void rewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + llvm_unreachable("must override rewrite or matchAndRewrite"); + } + virtual mlir::LogicalResult + matchAndRewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + if (mlir::failed(match(op))) + return mlir::failure(); + rewrite(op, adaptor, rewriter); + return mlir::success(); + } + +private: + using ConvertFIRToLLVMPattern::matchAndRewrite; + using ConvertToLLVMPattern::match; +}; + +/// FIR conversion pattern template +template +class FIROpAndTypeConversion : public FIROpConversion { +public: + using FIROpConversion::FIROpConversion; + using OpAdaptor = typename FromOp::Adaptor; + + mlir::LogicalResult + matchAndRewrite(FromOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const final { + mlir::Type ty = this->convertType(op.getType()); + return doRewrite(op, ty, adaptor, rewriter); + } + + virtual mlir::LogicalResult + doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const = 0; +}; + +} // namespace fir + +#endif // FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h index 396c13639255..79b3bfe4e80e 100644 --- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h +++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h @@ -94,8 +94,8 @@ public: // to LLVM IR dialect here. // // fir.complex | std.complex --> llvm<"{t,t}"> - template mlir::Type convertComplexType(C cmplx) const { - LLVM_DEBUG(llvm::dbgs() << "type convert: " << cmplx << '\n'); + template + mlir::Type convertComplexType(C cmplx) const { auto eleTy = cmplx.getElementType(); return convertType(specifics->complexMemoryType(eleTy)); } diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt index 175ab9fefda2..879bc28d017a 100644 --- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt +++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt @@ -3,6 +3,7 @@ add_flang_library(FIRCodeGen CGOps.cpp CodeGen.cpp CodeGenOpenMP.cpp + FIROpPatterns.cpp PreCGRewrite.cpp TBAABuilder.cpp Target.cpp diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index faf90ef6b50a..06ce84f1543a 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -14,6 +14,8 @@ #include "CGOps.h" #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h" +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "flang/Optimizer/CodeGen/TypeConverter.h" #include "flang/Optimizer/Dialect/FIRAttr.h" #include "flang/Optimizer/Dialect/FIROps.h" #include "flang/Optimizer/Dialect/FIRType.h" @@ -58,42 +60,14 @@ namespace fir { #define DEBUG_TYPE "flang-codegen" -// fir::LLVMTypeConverter for converting to LLVM IR dialect types. -#include "flang/Optimizer/CodeGen/TypeConverter.h" - // TODO: This should really be recovered from the specified target. static constexpr unsigned defaultAlign = 8; -static constexpr unsigned defaultAddressSpace = 0u; /// `fir.box` attribute values as defined for CFI_attribute_t in /// flang/ISO_Fortran_binding.h. static constexpr unsigned kAttrPointer = CFI_attribute_pointer; static constexpr unsigned kAttrAllocatable = CFI_attribute_allocatable; -static inline unsigned -getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getAllocaMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - -static inline unsigned -getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getProgramMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, unsigned addressSpace = 0) { return mlir::LLVM::LLVMPointerType::get(context, addressSpace); @@ -151,333 +125,9 @@ static unsigned getLenParamFieldId(mlir::Type ty) { return getTypeDescFieldId(ty) + 1; } -namespace { -/// FIR conversion pattern template -template -class FIROpConversion : public mlir::ConvertOpToLLVMPattern { -public: - explicit FIROpConversion(const fir::LLVMTypeConverter &lowering, - const fir::FIRToLLVMPassOptions &options) - : mlir::ConvertOpToLLVMPattern(lowering), options(options) {} - -protected: - mlir::Type convertType(mlir::Type ty) const { - return lowerTy().convertType(ty); - } - - // Convert FIR type to LLVM without turning fir.box into memory - // reference. - mlir::Type convertObjectType(mlir::Type firType) const { - if (auto boxTy = firType.dyn_cast()) - return lowerTy().convertBoxTypeAsStruct(boxTy); - return lowerTy().convertType(firType); - } - - mlir::LLVM::ConstantOp - genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, - int value) const { - mlir::Type i32Ty = rewriter.getI32Type(); - mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); - return rewriter.create(loc, i32Ty, attr); - } - - mlir::LLVM::ConstantOp - genConstantOffset(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - int offset) const { - mlir::Type ity = lowerTy().offsetType(); - mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); - return rewriter.create(loc, ity, cattr); - } - - /// Perform an extension or truncation as needed on an integer value. Lowering - /// to the specific target may involve some sign-extending or truncation of - /// values, particularly to fit them from abstract box types to the - /// appropriate reified structures. - mlir::Value integerCast(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - mlir::Type ty, mlir::Value val) const { - auto valTy = val.getType(); - // If the value was not yet lowered, lower its type so that it can - // be used in getPrimitiveTypeSizeInBits. - if (!valTy.isa()) - valTy = convertType(valTy); - auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); - auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); - if (toSize < fromSize) - return rewriter.create(loc, ty, val); - if (toSize > fromSize) - return rewriter.create(loc, ty, val); - return val; - } - - struct TypePair { - mlir::Type fir; - mlir::Type llvm; - }; - - TypePair getBoxTypePair(mlir::Type firBoxTy) const { - mlir::Type llvmBoxTy = lowerTy().convertBoxTypeAsStruct( - mlir::cast(firBoxTy)); - return TypePair{firBoxTy, llvmBoxTy}; - } - - /// Construct code sequence to extract the specific value from a `fir.box`. - mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, - mlir::Value box, mlir::Type resultTy, - mlir::ConversionPatternRewriter &rewriter, - int boxValue) const { - if (box.getType().isa()) { - auto pty = ::getLlvmPtrType(resultTy.getContext()); - auto p = rewriter.create( - loc, pty, boxTy.llvm, box, - llvm::ArrayRef{0, boxValue}); - auto loadOp = rewriter.create(loc, resultTy, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create(loc, box, boxValue); - } - - /// Method to construct code sequence to get the triple for dimension `dim` - /// from a box. - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, mlir::Value dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, int dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - mlir::Value - loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::Value dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - assert(box.getType().isa() && - "descriptor inquiry with runtime dim can only be done on descriptor " - "in memory"); - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - - mlir::Value - getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - int dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - if (box.getType().isa()) { - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create( - loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); - } - - mlir::Value - getStrideFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - unsigned dim, - mlir::ConversionPatternRewriter &rewriter) const { - auto idxTy = lowerTy().indexType(); - return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, - rewriter); - } - - /// Read base address from a fir.box. Returned address has type ty. - mlir::Value - getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); - } - - mlir::Value - getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, - kElemLenPosInBox); - } - - // Get the element type given an LLVM type that is of the form - // (array|struct|vector)+ and the provided indexes. - static mlir::Type getBoxEleTy(mlir::Type type, - llvm::ArrayRef indexes) { - for (unsigned i : indexes) { - if (auto t = type.dyn_cast()) { - assert(!t.isOpaque() && i < t.getBody().size()); - type = t.getBody()[i]; - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else { - fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), - "request for invalid box element type"); - } - } - return type; - } - - // Return LLVM type of the object described by a fir.box of \p boxType. - mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const { - mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); - assert(objectType && "boxType must be a box type"); - return this->convertType(objectType); - } - - /// Read the address of the type descriptor from a box. - mlir::Value - loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); - mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); - return getValueFromBox(loc, boxTy, box, tdescType, rewriter, - typeDescFieldId); - } - - // Load the attribute from the \p box and perform a check against \p maskValue - // The final comparison is implemented as `(attribute & maskValue) != 0`. - mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter, - unsigned maskValue) const { - mlir::Type attrTy = rewriter.getI32Type(); - mlir::Value attribute = - getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); - mlir::LLVM::ConstantOp attrMask = - genConstantOffset(loc, rewriter, maskValue); - auto maskRes = - rewriter.create(loc, attrTy, attribute, attrMask); - mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); - return rewriter.create( - loc, mlir::LLVM::ICmpPredicate::ne, maskRes, c0); - } - - template - mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter, - mlir::Value base, ARGS... args) const { - llvm::SmallVector cv = {args...}; - auto llvmPtrTy = ::getLlvmPtrType(ty.getContext()); - return rewriter.create(loc, llvmPtrTy, ty, base, cv); - } - - // Find the Block in which the alloca should be inserted. - // The order to recursively find the proper block: - // 1. An OpenMP Op that will be outlined. - // 2. A LLVMFuncOp - // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp - static mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) { - if (auto iface = - mlir::dyn_cast(op)) - return iface.getAllocaBlock(); - if (auto llvmFuncOp = mlir::dyn_cast(op)) - return &llvmFuncOp.front(); - return getBlockForAllocaInsert(op->getParentOp()); - } - - // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the - // allocation address space provided for the architecture in the DataLayout - // specification. If the address space is different from the devices - // program address space we perform a cast. In the case of most architectures - // the program and allocation address space will be the default of 0 and no - // cast will be emitted. - mlir::Value genAllocaAndAddrCastWithType( - mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, - mlir::ConversionPatternRewriter &rewriter) const { - auto thisPt = rewriter.saveInsertionPoint(); - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - if (mlir::isa(parentOp)) { - // DeclareReductionOp has multiple child regions. We want to get the first - // block of whichever of those regions we are currently in - mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); - rewriter.setInsertionPointToStart(&parentRegion->front()); - } else { - mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); - rewriter.setInsertionPointToStart(insertBlock); - } - auto size = genI32Constant(loc, rewriter, 1); - unsigned allocaAs = getAllocaAddressSpace(rewriter); - unsigned programAs = getProgramAddressSpace(rewriter); - - mlir::Value al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), - llvmObjectTy, size, alignment); - - // if our allocation address space, is not the same as the program address - // space, then we must emit a cast to the program address space before use. - // An example case would be on AMDGPU, where the allocation address space is - // the numeric value 5 (private), and the program address space is 0 - // (generic). - if (allocaAs != programAs) { - al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); - } - - rewriter.restoreInsertionPoint(thisPt); - return al; - } - - const fir::LLVMTypeConverter &lowerTy() const { - return *static_cast( - this->getTypeConverter()); - } - - void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, - mlir::Type baseFIRType, mlir::Type accessFIRType, - mlir::LLVM::GEPOp gep) const { - lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); - } - - const fir::FIRToLLVMPassOptions &options; -}; - -/// FIR conversion pattern template -template -class FIROpAndTypeConversion : public FIROpConversion { -public: - using FIROpConversion::FIROpConversion; - using OpAdaptor = typename FromOp::Adaptor; - - mlir::LogicalResult - matchAndRewrite(FromOp op, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const final { - mlir::Type ty = this->convertType(op.getType()); - return doRewrite(op, ty, adaptor, rewriter); - } - - virtual mlir::LogicalResult - doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const = 0; -}; -} // namespace - namespace { /// Lower `fir.address_of` operation to `llvm.address_of` operation. -struct AddrOfOpConversion : public FIROpConversion { +struct AddrOfOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -539,7 +189,7 @@ genAllocationScaleSize(OP op, mlir::Type ity, namespace { /// convert to LLVM IR dialect `alloca` -struct AllocaOpConversion : public FIROpConversion { +struct AllocaOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -624,7 +274,7 @@ struct AllocaOpConversion : public FIROpConversion { namespace { /// Lower `fir.box_addr` to the sequence of operations to extract the first /// element of the box. -struct BoxAddrOpConversion : public FIROpConversion { +struct BoxAddrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -645,7 +295,7 @@ struct BoxAddrOpConversion : public FIROpConversion { /// Convert `!fir.boxchar_len` to `!llvm.extractvalue` for the 2nd part of the /// boxchar. -struct BoxCharLenOpConversion : public FIROpConversion { +struct BoxCharLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -668,7 +318,7 @@ struct BoxCharLenOpConversion : public FIROpConversion { /// Lower `fir.box_dims` to a sequence of operations to extract the requested /// dimension information from the boxed value. /// Result in a triple set of GEPs and loads. -struct BoxDimsOpConversion : public FIROpConversion { +struct BoxDimsOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -690,7 +340,7 @@ struct BoxDimsOpConversion : public FIROpConversion { /// Lower `fir.box_elesize` to a sequence of operations ro extract the size of /// an element in the boxed value. -struct BoxEleSizeOpConversion : public FIROpConversion { +struct BoxEleSizeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -708,7 +358,7 @@ struct BoxEleSizeOpConversion : public FIROpConversion { /// Lower `fir.box_isalloc` to a sequence of operations to determine if the /// boxed value was from an ALLOCATABLE entity. -struct BoxIsAllocOpConversion : public FIROpConversion { +struct BoxIsAllocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -726,7 +376,7 @@ struct BoxIsAllocOpConversion : public FIROpConversion { /// Lower `fir.box_isarray` to a sequence of operations to determine if the /// boxed is an array. -struct BoxIsArrayOpConversion : public FIROpConversion { +struct BoxIsArrayOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -746,7 +396,7 @@ struct BoxIsArrayOpConversion : public FIROpConversion { /// Lower `fir.box_isptr` to a sequence of operations to determined if the /// boxed value was from a POINTER entity. -struct BoxIsPtrOpConversion : public FIROpConversion { +struct BoxIsPtrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -764,7 +414,7 @@ struct BoxIsPtrOpConversion : public FIROpConversion { /// Lower `fir.box_rank` to the sequence of operation to extract the rank from /// the box. -struct BoxRankOpConversion : public FIROpConversion { +struct BoxRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -784,7 +434,8 @@ struct BoxRankOpConversion : public FIROpConversion { /// Lower `fir.boxproc_host` operation. Extracts the host pointer from the /// boxproc. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct BoxProcHostOpConversion : public FIROpConversion { +struct BoxProcHostOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -797,7 +448,8 @@ struct BoxProcHostOpConversion : public FIROpConversion { /// Lower `fir.box_tdesc` to the sequence of operations to extract the type /// descriptor from the box. -struct BoxTypeDescOpConversion : public FIROpConversion { +struct BoxTypeDescOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -814,7 +466,8 @@ struct BoxTypeDescOpConversion : public FIROpConversion { /// Lower `fir.box_typecode` to a sequence of operations to extract the type /// code in the boxed value. -struct BoxTypeCodeOpConversion : public FIROpConversion { +struct BoxTypeCodeOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -832,7 +485,7 @@ struct BoxTypeCodeOpConversion : public FIROpConversion { }; /// Lower `fir.string_lit` to LLVM IR dialect operation. -struct StringLitOpConversion : public FIROpConversion { +struct StringLitOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -872,7 +525,7 @@ struct StringLitOpConversion : public FIROpConversion { }; /// `fir.call` -> `llvm.call` -struct CallOpConversion : public FIROpConversion { +struct CallOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -903,7 +556,7 @@ namespace { /// Per 10.1, the only comparisons available are .EQ. (oeq) and .NE. (une). /// /// For completeness, all other comparison are done on the real component only. -struct CmpcOpConversion : public FIROpConversion { +struct CmpcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -941,7 +594,7 @@ struct CmpcOpConversion : public FIROpConversion { }; /// Lower complex constants -struct ConstcOpConversion : public FIROpConversion { +struct ConstcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -968,7 +621,7 @@ struct ConstcOpConversion : public FIROpConversion { }; /// convert value of from-type to value of to-type -struct ConvertOpConversion : public FIROpConversion { +struct ConvertOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; static bool isFloatingPointTy(mlir::Type ty) { @@ -1137,7 +790,7 @@ struct ConvertOpConversion : public FIROpConversion { /// only used to carry information during FIR to FIR passes. It may be used /// in the future to generate the runtime type info data structures instead /// of generating them in lowering. -struct TypeInfoOpConversion : public FIROpConversion { +struct TypeInfoOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1150,7 +803,7 @@ struct TypeInfoOpConversion : public FIROpConversion { /// `fir.dt_entry` operation has no specific CodeGen. The operation is only used /// to carry information during FIR to FIR passes. -struct DTEntryOpConversion : public FIROpConversion { +struct DTEntryOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1162,7 +815,7 @@ struct DTEntryOpConversion : public FIROpConversion { }; /// Lower `fir.global_len` operation. -struct GlobalLenOpConversion : public FIROpConversion { +struct GlobalLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1175,7 +828,7 @@ struct GlobalLenOpConversion : public FIROpConversion { /// Lower fir.len_param_index struct LenParamIndexOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // FIXME: this should be specialized by the runtime target @@ -1190,7 +843,7 @@ struct LenParamIndexOpConversion /// instructions that generate `!llvm.struct<(ptr, i64)>`. The 1st element /// in this struct is a pointer. Its type is determined from `KIND`. The 2nd /// element is the length of the character buffer (`#n`). -struct EmboxCharOpConversion : public FIROpConversion { +struct EmboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1283,7 +936,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy, namespace { /// Lower a `fir.allocmem` instruction into `llvm.call @malloc` -struct AllocMemOpConversion : public FIROpConversion { +struct AllocMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1352,7 +1005,7 @@ static unsigned getDimension(mlir::LLVM::LLVMArrayType ty) { namespace { /// Lower a `fir.freemem` instruction into `llvm.call @free` -struct FreeMemOpConversion : public FIROpConversion { +struct FreeMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1410,9 +1063,9 @@ convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy, /// Common base class for embox to descriptor conversion. template -struct EmboxCommonConversion : public FIROpConversion { - using FIROpConversion::FIROpConversion; - using TypePair = typename FIROpConversion::TypePair; +struct EmboxCommonConversion : public fir::FIROpConversion { + using fir::FIROpConversion::FIROpConversion; + using TypePair = typename fir::FIROpConversion::TypePair; static int getCFIAttr(fir::BaseBoxType boxTy) { auto eleTy = boxTy.getEleTy(); @@ -1434,8 +1087,8 @@ struct EmboxCommonConversion : public FIROpConversion { return size; // Length accounted for in the genTypeStrideInBytes GEP. // Otherwise, multiply the single character size by the length. assert(!lenParams.empty()); - auto len64 = FIROpConversion::integerCast(loc, rewriter, i64Ty, - lenParams.back()); + auto len64 = fir::FIROpConversion::integerCast(loc, rewriter, i64Ty, + lenParams.back()); return rewriter.create(loc, i64Ty, size, len64); } @@ -2280,7 +1933,7 @@ private: /// Lower `fir.emboxproc` operation. Creates a procedure box. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct EmboxProcOpConversion : public FIROpConversion { +struct EmboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2346,7 +1999,7 @@ private: namespace { /// Extract a subobject value from an ssa-value of aggregate type struct ExtractValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2365,7 +2018,7 @@ struct ExtractValueOpConversion /// InsertValue is the generalized instruction for the composition of new /// aggregate type values. struct InsertValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2383,7 +2036,7 @@ struct InsertValueOpConversion /// InsertOnRange inserts a value into a sequence over a range of offsets. struct InsertOnRangeOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; // Increments an array of subscripts in a row major fasion. @@ -2447,7 +2100,7 @@ namespace { /// (See the static restriction on coordinate_of.) array_coor determines the /// coordinate (location) of a specific element. struct XArrayCoorOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2615,7 +2268,7 @@ struct XArrayCoorOpConversion /// With unboxed arrays, there is the restriction that the array have a static /// shape in all but the last column. struct CoordinateOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2910,7 +2563,7 @@ private: /// Convert `fir.field_index`. The conversion depends on whether the size of /// the record is static or dynamic. -struct FieldIndexOpConversion : public FIROpConversion { +struct FieldIndexOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // NB: most field references should be resolved by this point @@ -2951,7 +2604,7 @@ struct FieldIndexOpConversion : public FIROpConversion { }; /// Convert `fir.end` -struct FirEndOpConversion : public FIROpConversion { +struct FirEndOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2963,7 +2616,7 @@ struct FirEndOpConversion : public FIROpConversion { }; /// Lower `fir.type_desc` to a global addr. -struct TypeDescOpConversion : public FIROpConversion { +struct TypeDescOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2990,7 +2643,7 @@ struct TypeDescOpConversion : public FIROpConversion { }; /// Lower `fir.has_value` operation to `llvm.return` operation. -struct HasValueOpConversion : public FIROpConversion { +struct HasValueOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3043,7 +2696,7 @@ static inline bool attributeTypeIsCompatible(mlir::MLIRContext *ctx, /// Lower `fir.global` operation to `llvm.global` operation. /// `fir.insert_on_range` operations are replaced with constant dense attribute /// if they are applied on the full range. -struct GlobalOpConversion : public FIROpConversion { +struct GlobalOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3167,7 +2820,7 @@ private: }; /// `fir.load` --> `llvm.load` -struct LoadOpConversion : public FIROpConversion { +struct LoadOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3216,7 +2869,7 @@ struct LoadOpConversion : public FIROpConversion { /// Lower `fir.no_reassoc` to LLVM IR dialect. /// TODO: how do we want to enforce this in LLVM-IR? Can we manipulate the fast /// math flags? -struct NoReassocOpConversion : public FIROpConversion { +struct NoReassocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3277,7 +2930,7 @@ static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp, /// upper bound in the same case condition. /// /// TODO: lowering of CHARACTER type cases is not handled yet. -struct SelectCaseOpConversion : public FIROpConversion { +struct SelectCaseOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3391,7 +3044,7 @@ static void selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering, } /// conversion of fir::SelectOp to an if-then-else ladder -struct SelectOpConversion : public FIROpConversion { +struct SelectOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3403,7 +3056,7 @@ struct SelectOpConversion : public FIROpConversion { }; /// conversion of fir::SelectRankOp to an if-then-else ladder -struct SelectRankOpConversion : public FIROpConversion { +struct SelectRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3415,7 +3068,7 @@ struct SelectRankOpConversion : public FIROpConversion { }; /// Lower `fir.select_type` to LLVM IR dialect. -struct SelectTypeOpConversion : public FIROpConversion { +struct SelectTypeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3428,7 +3081,7 @@ struct SelectTypeOpConversion : public FIROpConversion { }; /// `fir.store` --> `llvm.store` -struct StoreOpConversion : public FIROpConversion { +struct StoreOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3462,7 +3115,7 @@ namespace { /// Convert `fir.unboxchar` into two `llvm.extractvalue` instructions. One for /// the character buffer and one for the buffer length. -struct UnboxCharOpConversion : public FIROpConversion { +struct UnboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3487,7 +3140,7 @@ struct UnboxCharOpConversion : public FIROpConversion { /// Lower `fir.unboxproc` operation. Unbox a procedure box value, yielding its /// components. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct UnboxProcOpConversion : public FIROpConversion { +struct UnboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3499,7 +3152,7 @@ struct UnboxProcOpConversion : public FIROpConversion { }; /// convert to LLVM IR dialect `undef` -struct UndefOpConversion : public FIROpConversion { +struct UndefOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3511,7 +3164,7 @@ struct UndefOpConversion : public FIROpConversion { } }; -struct ZeroOpConversion : public FIROpConversion { +struct ZeroOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3524,7 +3177,8 @@ struct ZeroOpConversion : public FIROpConversion { }; /// `fir.unreachable` --> `llvm.unreachable` -struct UnreachableOpConversion : public FIROpConversion { +struct UnreachableOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3541,7 +3195,7 @@ struct UnreachableOpConversion : public FIROpConversion { /// %1 = llvm.ptrtoint %0 /// %2 = llvm.icmp "ne" %1, %0 : i64 /// ``` -struct IsPresentOpConversion : public FIROpConversion { +struct IsPresentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3570,7 +3224,7 @@ struct IsPresentOpConversion : public FIROpConversion { /// Create value signaling an absent optional argument in a call, e.g. /// `fir.absent !fir.ref` --> `llvm.mlir.zero : !llvm.ptr` -struct AbsentOpConversion : public FIROpConversion { +struct AbsentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3630,7 +3284,7 @@ complexSum(OPTY sumop, mlir::ValueRange opnds, } // namespace namespace { -struct AddcOpConversion : public FIROpConversion { +struct AddcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3645,7 +3299,7 @@ struct AddcOpConversion : public FIROpConversion { } }; -struct SubcOpConversion : public FIROpConversion { +struct SubcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3661,7 +3315,7 @@ struct SubcOpConversion : public FIROpConversion { }; /// Inlined complex multiply -struct MulcOpConversion : public FIROpConversion { +struct MulcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3695,7 +3349,7 @@ struct MulcOpConversion : public FIROpConversion { }; /// Inlined complex division -struct DivcOpConversion : public FIROpConversion { +struct DivcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3735,7 +3389,7 @@ struct DivcOpConversion : public FIROpConversion { }; /// Inlined complex negation -struct NegcOpConversion : public FIROpConversion { +struct NegcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3756,7 +3410,7 @@ struct NegcOpConversion : public FIROpConversion { } }; -struct BoxOffsetOpConversion : public FIROpConversion { +struct BoxOffsetOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3782,10 +3436,10 @@ struct BoxOffsetOpConversion : public FIROpConversion { /// anymore uses. /// These operations are normally dead after the pre-codegen pass. template -struct MustBeDeadConversion : public FIROpConversion { +struct MustBeDeadConversion : public fir::FIROpConversion { explicit MustBeDeadConversion(const fir::LLVMTypeConverter &lowering, const fir::FIRToLLVMPassOptions &options) - : FIROpConversion(lowering, options) {} + : fir::FIROpConversion(lowering, options) {} using OpAdaptor = typename FromOp::Adaptor; mlir::LogicalResult @@ -3799,7 +3453,7 @@ struct MustBeDeadConversion : public FIROpConversion { }; struct UnrealizedConversionCastOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp new file mode 100644 index 000000000000..26871d888815 --- /dev/null +++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp @@ -0,0 +1,315 @@ +//===-- CodeGen.cpp -- bridge to lower to LLVM ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ +// +//===----------------------------------------------------------------------===// + +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "mlir/Dialect/OpenMP/OpenMPDialect.h" +#include "llvm/Support/Debug.h" + +static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, + unsigned addressSpace = 0) { + return mlir::LLVM::LLVMPointerType::get(context, addressSpace); +} + +static unsigned getTypeDescFieldId(mlir::Type ty) { + auto isArray = fir::dyn_cast_ptrOrBoxEleTy(ty).isa(); + return isArray ? kOptTypePtrPosInBox : kDimsPosInBox; +} + +namespace fir { + +ConvertFIRToLLVMPattern::ConvertFIRToLLVMPattern( + llvm::StringRef rootOpName, mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, mlir::PatternBenefit benefit) + : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit), + options(options) {} + +// Convert FIR type to LLVM without turning fir.box into memory +// reference. +mlir::Type +ConvertFIRToLLVMPattern::convertObjectType(mlir::Type firType) const { + if (auto boxTy = firType.dyn_cast()) + return lowerTy().convertBoxTypeAsStruct(boxTy); + return lowerTy().convertType(firType); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genI32Constant( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const { + mlir::Type i32Ty = rewriter.getI32Type(); + mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); + return rewriter.create(loc, i32Ty, attr); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int offset) const { + mlir::Type ity = lowerTy().offsetType(); + mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); + return rewriter.create(loc, ity, cattr); +} + +/// Perform an extension or truncation as needed on an integer value. Lowering +/// to the specific target may involve some sign-extending or truncation of +/// values, particularly to fit them from abstract box types to the +/// appropriate reified structures. +mlir::Value +ConvertFIRToLLVMPattern::integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const { + auto valTy = val.getType(); + // If the value was not yet lowered, lower its type so that it can + // be used in getPrimitiveTypeSizeInBits. + if (!valTy.isa()) + valTy = convertType(valTy); + auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); + auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); + if (toSize < fromSize) + return rewriter.create(loc, ty, val); + if (toSize > fromSize) + return rewriter.create(loc, ty, val); + return val; +} + +fir::ConvertFIRToLLVMPattern::TypePair +ConvertFIRToLLVMPattern::getBoxTypePair(mlir::Type firBoxTy) const { + mlir::Type llvmBoxTy = + lowerTy().convertBoxTypeAsStruct(mlir::cast(firBoxTy)); + return TypePair{firBoxTy, llvmBoxTy}; +} + +/// Construct code sequence to extract the specific value from a `fir.box`. +mlir::Value ConvertFIRToLLVMPattern::getValueFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, int boxValue) const { + if (box.getType().isa()) { + auto pty = getLlvmPtrType(resultTy.getContext()); + auto p = rewriter.create( + loc, pty, boxTy.llvm, box, + llvm::ArrayRef{0, boxValue}); + auto loadOp = rewriter.create(loc, resultTy, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create(loc, box, boxValue); +} + +/// Method to construct code sequence to get the triple for dimension `dim` +/// from a box. +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, int dim, mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +mlir::Value ConvertFIRToLLVMPattern::loadDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Value dim, + int off, mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + assert(box.getType().isa() && + "descriptor inquiry with runtime dim can only be done on descriptor " + "in memory"); + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; +} + +mlir::Value ConvertFIRToLLVMPattern::getDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, int dim, int off, + mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + if (box.getType().isa()) { + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create( + loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); +} + +mlir::Value ConvertFIRToLLVMPattern::getStrideFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const { + auto idxTy = lowerTy().indexType(); + return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, + rewriter); +} + +/// Read base address from a fir.box. Returned address has type ty. +mlir::Value ConvertFIRToLLVMPattern::getBaseAddrFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); +} + +mlir::Value ConvertFIRToLLVMPattern::getElementSizeFromBox( + mlir::Location loc, mlir::Type resultTy, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kElemLenPosInBox); +} + +// Get the element type given an LLVM type that is of the form +// (array|struct|vector)+ and the provided indexes. +mlir::Type ConvertFIRToLLVMPattern::getBoxEleTy( + mlir::Type type, llvm::ArrayRef indexes) const { + for (unsigned i : indexes) { + if (auto t = type.dyn_cast()) { + assert(!t.isOpaque() && i < t.getBody().size()); + type = t.getBody()[i]; + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else { + fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), + "request for invalid box element type"); + } + } + return type; +} + +// Return LLVM type of the object described by a fir.box of \p boxType. +mlir::Type ConvertFIRToLLVMPattern::getLlvmObjectTypeFromBoxType( + mlir::Type boxType) const { + mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); + assert(objectType && "boxType must be a box type"); + return this->convertType(objectType); +} + +/// Read the address of the type descriptor from a box. +mlir::Value ConvertFIRToLLVMPattern::loadTypeDescAddress( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); + mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); + return getValueFromBox(loc, boxTy, box, tdescType, rewriter, typeDescFieldId); +} + +// Load the attribute from the \p box and perform a check against \p maskValue +// The final comparison is implemented as `(attribute & maskValue) != 0`. +mlir::Value ConvertFIRToLLVMPattern::genBoxAttributeCheck( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, unsigned maskValue) const { + mlir::Type attrTy = rewriter.getI32Type(); + mlir::Value attribute = + getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); + mlir::LLVM::ConstantOp attrMask = genConstantOffset(loc, rewriter, maskValue); + auto maskRes = + rewriter.create(loc, attrTy, attribute, attrMask); + mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); + return rewriter.create(loc, mlir::LLVM::ICmpPredicate::ne, + maskRes, c0); +} + +// Find the Block in which the alloca should be inserted. +// The order to recursively find the proper block: +// 1. An OpenMP Op that will be outlined. +// 2. A LLVMFuncOp +// 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp +mlir::Block * +ConvertFIRToLLVMPattern::getBlockForAllocaInsert(mlir::Operation *op) const { + if (auto iface = mlir::dyn_cast(op)) + return iface.getAllocaBlock(); + if (auto llvmFuncOp = mlir::dyn_cast(op)) + return &llvmFuncOp.front(); + return getBlockForAllocaInsert(op->getParentOp()); +} + +// Generate an alloca of size 1 for an object of type \p llvmObjectTy in the +// allocation address space provided for the architecture in the DataLayout +// specification. If the address space is different from the devices +// program address space we perform a cast. In the case of most architectures +// the program and allocation address space will be the default of 0 and no +// cast will be emitted. +mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType( + mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const { + auto thisPt = rewriter.saveInsertionPoint(); + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + if (mlir::isa(parentOp)) { + // DeclareReductionOp has multiple child regions. We want to get the first + // block of whichever of those regions we are currently in + mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); + rewriter.setInsertionPointToStart(&parentRegion->front()); + } else { + mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); + rewriter.setInsertionPointToStart(insertBlock); + } + auto size = genI32Constant(loc, rewriter, 1); + unsigned allocaAs = getAllocaAddressSpace(rewriter); + unsigned programAs = getProgramAddressSpace(rewriter); + + mlir::Value al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), llvmObjectTy, + size, alignment); + + // if our allocation address space, is not the same as the program address + // space, then we must emit a cast to the program address space before use. + // An example case would be on AMDGPU, where the allocation address space is + // the numeric value 5 (private), and the program address space is 0 + // (generic). + if (allocaAs != programAs) { + al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); + } + + rewriter.restoreInsertionPoint(thisPt); + return al; +} + +unsigned ConvertFIRToLLVMPattern::getAllocaAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getAllocaMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getProgramMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +} // namespace fir -- GitLab From 19268ac55106834701b1c41acded4d413a3502e9 Mon Sep 17 00:00:00 2001 From: Mariusz Borsa Date: Fri, 22 Mar 2024 13:36:17 -0700 Subject: [PATCH 207/933] [Sanitizers][Darwin] Bump up DEFAULT_SANITIZER_MIN_OSX_VERSION (#86035) The greendragon was recently moved and now it runs on somewhat newer macOS version - which breaks some sanitizers tests rdar://125052915 Co-authored-by: Mariusz Borsa --- compiler-rt/cmake/config-ix.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 4f47142850a5..46a6fdf8728f 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -461,7 +461,7 @@ if(APPLE) set(ORC_SUPPORTED_OS osx) endif() - set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.10) + set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.13) set(DARWIN_osx_MIN_VER_FLAG "-mmacosx-version-min") if(NOT SANITIZER_MIN_OSX_VERSION) string(REGEX MATCH "${DARWIN_osx_MIN_VER_FLAG}=([.0-9]+)" -- GitLab From 105feb9ac61243a32c84f5c13015602e5de500ff Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:41:27 -0700 Subject: [PATCH 208/933] [BOLT][DWARF] Fix handling of DW_TAG_label (#86182) For DWARF5 BOLT was not retreiving address and instead was setting an index. Changed so that an address is used, and added DWARF4 test because it was missing. --- bolt/lib/Rewrite/DWARFRewriter.cpp | 43 +++-- bolt/test/X86/dwarf4-label-low-pc.s | 263 ++++++++++++++++++++++++++++ bolt/test/X86/dwarf5-label-low-pc.s | 36 ++-- 3 files changed, 316 insertions(+), 26 deletions(-) create mode 100644 bolt/test/X86/dwarf4-label-low-pc.s diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index 8dc7b90a6e30..601a2105fc26 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -375,12 +375,11 @@ static cl::opt AlwaysConvertToRanges( extern cl::opt CompDirOverride; } // namespace opts -static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, - uint64_t &LowPC, uint64_t &HighPC, - uint64_t &SectionIndex) { +/// If DW_AT_low_pc exists sets LowPC and returns true. +static bool getLowPC(const DIE &Die, const DWARFUnit &DU, uint64_t &LowPC, + uint64_t &SectionIndex) { DIEValue DvalLowPc = Die.findAttribute(dwarf::DW_AT_low_pc); - DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc); - if (!DvalLowPc || !DvalHighPc) + if (!DvalLowPc) return false; dwarf::Form Form = DvalLowPc.getForm(); @@ -403,14 +402,39 @@ static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, LowPC = LowPcValue; SectionIndex = 0; } + return true; +} + +/// If DW_AT_high_pc exists sets HighPC and returns true. +static bool getHighPC(const DIE &Die, const uint64_t LowPC, uint64_t &HighPC) { + DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc); + if (!DvalHighPc) + return false; if (DvalHighPc.getForm() == dwarf::DW_FORM_addr) HighPC = DvalHighPc.getDIEInteger().getValue(); else HighPC = LowPC + DvalHighPc.getDIEInteger().getValue(); - return true; } +/// If DW_AT_low_pc and DW_AT_high_pc exist sets LowPC and HighPC and returns +/// true. +static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, + uint64_t &LowPC, uint64_t &HighPC, + uint64_t &SectionIndex) { + uint64_t TempLowPC = LowPC; + uint64_t TempHighPC = HighPC; + uint64_t TempSectionIndex = SectionIndex; + if (getLowPC(Die, DU, TempLowPC, TempSectionIndex) && + getHighPC(Die, TempLowPC, TempHighPC)) { + LowPC = TempLowPC; + HighPC = TempHighPC; + SectionIndex = TempSectionIndex; + return true; + } + return false; +} + static Expected getDIEAddressRanges(const DIE &Die, DWARFUnit &DU) { uint64_t LowPC, HighPC, Index; @@ -1248,10 +1272,9 @@ void DWARFRewriter::updateUnitDebugInfo( } } } else if (LowPCAttrInfo) { - const std::optional Result = - LowPCAttrInfo.getDIEInteger().getValue(); - if (Result.has_value()) { - const uint64_t Address = Result.value(); + uint64_t Address = 0; + uint64_t SectionIndex = 0; + if (getLowPC(*Die, Unit, Address, SectionIndex)) { uint64_t NewAddress = 0; if (const BinaryFunction *Function = BC.getBinaryFunctionContainingAddress(Address)) { diff --git a/bolt/test/X86/dwarf4-label-low-pc.s b/bolt/test/X86/dwarf4-label-low-pc.s new file mode 100644 index 000000000000..dfd5af18c09b --- /dev/null +++ b/bolt/test/X86/dwarf4-label-low-pc.s @@ -0,0 +1,263 @@ + +# REQUIRES: system-linux + +# RUN: llvm-mc -dwarf-version=4 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o +# RUN: %clang %cflags -dwarf-4 %tmain.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections +# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.exe | FileCheck --check-prefix=PRECHECK %s +# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt > %t.txt +# RUN: llvm-objdump -d %t.bolt >> %t.txt +# RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s + +## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addr] that is part of DW_TAG_label. + +# PRECHECK: version = 0x0004 +# PRECHECK: DW_TAG_label +# PRECHECK-NEXT: DW_AT_name +# PRECHECK-NEXT: DW_AT_decl_file +# PRECHECK-NEXT: DW_AT_decl_line +# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] +# PRECHECK: DW_TAG_label +# PRECHECK-NEXT: DW_AT_name +# PRECHECK-NEXT: DW_AT_decl_file +# PRECHECK-NEXT: DW_AT_decl_line +# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] + +# POSTCHECK: version = 0x0004 +# POSTCHECK: DW_TAG_label +# POSTCHECK-NEXT: DW_AT_name +# POSTCHECK-NEXT: DW_AT_decl_file +# POSTCHECK-NEXT: DW_AT_decl_line +# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR:[1-9a-f]*]] +# POSTCHECK: DW_TAG_label +# POSTCHECK-NEXT: DW_AT_name +# POSTCHECK-NEXT: DW_AT_decl_file +# POSTCHECK-NEXT: DW_AT_decl_line +# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR2:[1-9a-f]*]] + +# POSTCHECK: [[ADDR]]: 8b 45 f8 +# POSTCHECK: [[ADDR2]]: 8b 45 f8 + +## clang++ main.cpp -g2 -gdwarf-4 -S +## int main() { +## int a = 4; +## if (a == 5) +## goto LABEL1; +## else +## goto LABEL2; +## LABEL1:a++; +## LABEL2:a--; +## return 0; +## } + + .text + .file "main.cpp" + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .file 1 "/home" "main.cpp" + .loc 1 1 0 # main.cpp:1:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl $0, -4(%rbp) +.Ltmp0: + .loc 1 2 7 prologue_end # main.cpp:2:7 + movl $4, -8(%rbp) +.Ltmp1: + .loc 1 3 9 # main.cpp:3:9 + cmpl $5, -8(%rbp) +.Ltmp2: + .loc 1 3 7 is_stmt 0 # main.cpp:3:7 + jne .LBB0_2 +# %bb.1: # %if.then +.Ltmp3: + .loc 1 4 5 is_stmt 1 # main.cpp:4:5 + jmp .LBB0_3 +.LBB0_2: # %if.else + .loc 1 6 5 # main.cpp:6:5 + jmp .LBB0_4 +.Ltmp4: +.LBB0_3: # %LABEL1 + #DEBUG_LABEL: main:LABEL1 + .loc 1 7 11 # main.cpp:7:11 + movl -8(%rbp), %eax + addl $1, %eax + movl %eax, -8(%rbp) +.LBB0_4: # %LABEL2 +.Ltmp5: + #DEBUG_LABEL: main:LABEL2 + .loc 1 8 11 # main.cpp:8:11 + movl -8(%rbp), %eax + addl $-1, %eax + movl %eax, -8(%rbp) + .loc 1 9 3 # main.cpp:9:3 + xorl %eax, %eax + .loc 1 9 3 epilogue_begin is_stmt 0 # main.cpp:9:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp6: +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + # -- End function + .section .debug_abbrev,"",@progbits + .byte 1 # Abbreviation Code + .byte 17 # DW_TAG_compile_unit + .byte 1 # DW_CHILDREN_yes + .byte 37 # DW_AT_producer + .byte 14 # DW_FORM_strp + .byte 19 # DW_AT_language + .byte 5 # DW_FORM_data2 + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 16 # DW_AT_stmt_list + .byte 23 # DW_FORM_sec_offset + .byte 27 # DW_AT_comp_dir + .byte 14 # DW_FORM_strp + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 2 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 3 # Abbreviation Code + .byte 52 # DW_TAG_variable + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 4 # Abbreviation Code + .byte 10 # DW_TAG_label + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 5 # Abbreviation Code + .byte 36 # DW_TAG_base_type + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 62 # DW_AT_encoding + .byte 11 # DW_FORM_data1 + .byte 11 # DW_AT_byte_size + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 0 # EOM(3) + .section .debug_info,"",@progbits +.Lcu_begin0: + .long .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit +.Ldebug_info_start0: + .short 4 # DWARF version number + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 8 # Address Size (in bytes) + .byte 1 # Abbrev [1] 0xb:0x6d DW_TAG_compile_unit + .long .Linfo_string0 # DW_AT_producer + .short 33 # DW_AT_language + .long .Linfo_string1 # DW_AT_name + .long .Lline_table_start0 # DW_AT_stmt_list + .long .Linfo_string2 # DW_AT_comp_dir + .quad .Lfunc_begin0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 2 # Abbrev [2] 0x2a:0x46 DW_TAG_subprogram + .quad .Lfunc_begin0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .long .Linfo_string3 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 112 # DW_AT_type + # DW_AT_external + .byte 3 # Abbrev [3] 0x43:0xe DW_TAG_variable + .byte 2 # DW_AT_location + .byte 145 + .byte 120 + .long .Linfo_string5 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long 112 # DW_AT_type + .byte 4 # Abbrev [4] 0x51:0xf DW_TAG_label + .long .Linfo_string6 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 7 # DW_AT_decl_line + .quad .Ltmp4 # DW_AT_low_pc + .byte 4 # Abbrev [4] 0x60:0xf DW_TAG_label + .long .Linfo_string7 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 8 # DW_AT_decl_line + .quad .Ltmp5 # DW_AT_low_pc + .byte 0 # End Of Children Mark + .byte 5 # Abbrev [5] 0x70:0x7 DW_TAG_base_type + .long .Linfo_string4 # DW_AT_name + .byte 5 # DW_AT_encoding + .byte 4 # DW_AT_byte_size + .byte 0 # End Of Children Mark +.Ldebug_info_end0: + .section .debug_str,"MS",@progbits,1 +.Linfo_string0: + .asciz "clang version 19.0.0git" # string offset=0 +.Linfo_string1: + .asciz "main.cpp" # string offset=24 +.Linfo_string2: + .asciz "/home" # string offset=33 +.Linfo_string3: + .asciz "main" # string offset=71 +.Linfo_string4: + .asciz "int" # string offset=76 +.Linfo_string5: + .asciz "a" # string offset=80 +.Linfo_string6: + .asciz "LABEL1" # string offset=82 +.Linfo_string7: + .asciz "LABEL2" # string offset=89 + .ident "clang version 19.0.0git" + .section ".note.GNU-stack","",@progbits + .addrsig + .section .debug_line,"",@progbits +.Lline_table_start0: diff --git a/bolt/test/X86/dwarf5-label-low-pc.s b/bolt/test/X86/dwarf5-label-low-pc.s index 890d9e024d1a..1e3fc17ad516 100644 --- a/bolt/test/X86/dwarf5-label-low-pc.s +++ b/bolt/test/X86/dwarf5-label-low-pc.s @@ -8,9 +8,10 @@ # RUN: llvm-dwarfdump --show-form --verbose --debug-addr %t.bolt > %t.txt # RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt >> %t.txt +# RUN: llvm-objdump -d %t.bolt >> %t.txt # RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s -# This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label. +## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label. # PRECHECK: version = 0x0005 # PRECHECK: DW_TAG_label @@ -28,8 +29,8 @@ # POSTCHECK: Addrs: [ # POSTCHECK-NEXT: 0x # POSTCHECK-NEXT: 0x -# POSTCHECK-NEXT: 0x[[#%.16x,ADDR:]] -# POSTCHECK-NEXT: 0x[[#%.16x,ADDR2:]] +# POSTCHECK-NEXT: 0x[[ADDR:[1-9a-f]*]] +# POSTCHECK-NEXT: 0x[[ADDR2:[1-9a-f]*]] # POSTCHECK: version = 0x0005 # POSTCHECK: DW_TAG_label @@ -37,25 +38,28 @@ # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000002) -# POSTCHECK-SAME: 0x[[#ADDR]] +# POSTCHECK-SAME: 0x[[ADDR]] # POSTCHECK: DW_TAG_label # POSTCHECK-NEXT: DW_AT_name # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000003) -# POSTCHECK-SAME: 0x[[#ADDR2]] +# POSTCHECK-SAME: 0x[[ADDR2]] -# clang++ main.cpp -g -S -# int main() { -# int a = 4; -# if (a == 5) -# goto LABEL1; -# else -# goto LABEL2; -# LABEL1:a++; -# LABEL2:a--; -# return 0; -# } +# POSTCHECK: [[ADDR]]: 8b 45 f8 +# POSTCHECK: [[ADDR2]]: 8b 45 f8 + +## clang++ main.cpp -g -S +## int main() { +## int a = 4; +## if (a == 5) +## goto LABEL1; +## else +## goto LABEL2; +## LABEL1:a++; +## LABEL2:a--; +## return 0; +## } .text .file "main.cpp" -- GitLab From f3cfe016c5d8429c0dccfa6f85442e2ea0d45a58 Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:48:49 -0700 Subject: [PATCH 209/933] [BOLT][DWARF] Add support for cross-cu references for debug-names (#86015) The DW_AT_abstract_origin can be a cross-cu reference as a by-product of LTO. On IR level for absolute references an address is stored, vs a DIE for relative references. Added a map to keep track of cross-cu referenced DIEs to use when we add an Entry. --- bolt/include/bolt/Core/DebugNames.h | 11 + bolt/lib/Core/DIEBuilder.cpp | 6 +- bolt/lib/Core/DebugNames.cpp | 109 +-- bolt/test/X86/dwarf5-debug-names-cross-cu.s | 712 ++++++++++++++++++++ 4 files changed, 796 insertions(+), 42 deletions(-) create mode 100644 bolt/test/X86/dwarf5-debug-names-cross-cu.s diff --git a/bolt/include/bolt/Core/DebugNames.h b/bolt/include/bolt/Core/DebugNames.h index 1f17f1ae4d13..fbaa7f4e68aa 100644 --- a/bolt/include/bolt/Core/DebugNames.h +++ b/bolt/include/bolt/Core/DebugNames.h @@ -68,6 +68,16 @@ public: std::unique_ptr releaseBuffer() { return std::move(FullTableBuffer); } + /// Adds a DIE that is referenced across CUs. + void addCrossCUDie(const DIE *Die) { + CrossCUDies.insert({Die->getOffset(), Die}); + } + /// Returns true if the DIE can generate an entry for a cross cu reference. + /// This only checks TAGs of a DIE because when this is invoked DIE might not + /// be fully constructed. + bool canGenerateEntryWithCrossCUReference( + const DWARFUnit &Unit, const DIE &Die, + const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec); private: BinaryContext &BC; @@ -128,6 +138,7 @@ private: llvm::DenseMap CUOffsetsToPatch; // Contains a map of Entry ID to Entry relative offset. llvm::DenseMap EntryRelativeOffsets; + llvm::DenseMap CrossCUDies; /// Adds Unit to either CUList, LocalTUList or ForeignTUList. /// Input Unit being processed, and DWO ID if Unit is being processed comes /// from a DWO section. diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp index 0cf8a5e8c2c3..354fe5059443 100644 --- a/bolt/lib/Core/DIEBuilder.cpp +++ b/bolt/lib/Core/DIEBuilder.cpp @@ -545,6 +545,10 @@ void DIEBuilder::cloneDieReferenceAttribute( NewRefDie = DieInfo.Die; if (AttrSpec.Form == dwarf::DW_FORM_ref_addr) { + // Adding referenced DIE to DebugNames to be used when entries are created + // that contain cross cu references. + if (DebugNamesTable.canGenerateEntryWithCrossCUReference(U, Die, AttrSpec)) + DebugNamesTable.addCrossCUDie(DieInfo.Die); // no matter forward reference or backward reference, we are supposed // to calculate them in `finish` due to the possible modification of // the DIE. @@ -554,7 +558,7 @@ void DIEBuilder::cloneDieReferenceAttribute( std::make_pair(CurDieInfo, AddrReferenceInfo(&DieInfo, AttrSpec))); Die.addValue(getState().DIEAlloc, AttrSpec.Attr, dwarf::DW_FORM_ref_addr, - DIEInteger(0xDEADBEEF)); + DIEInteger(DieInfo.Die->getOffset())); return; } diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp index 23a29f52513c..049244c4b515 100644 --- a/bolt/lib/Core/DebugNames.cpp +++ b/bolt/lib/Core/DebugNames.cpp @@ -146,6 +146,55 @@ static bool shouldIncludeVariable(const DWARFUnit &Unit, const DIE &Die) { return false; } +bool static canProcess(const DWARFUnit &Unit, const DIE &Die, + std::string &NameToUse, const bool TagsOnly) { + switch (Die.getTag()) { + case dwarf::DW_TAG_base_type: + case dwarf::DW_TAG_class_type: + case dwarf::DW_TAG_enumeration_type: + case dwarf::DW_TAG_imported_declaration: + case dwarf::DW_TAG_pointer_type: + case dwarf::DW_TAG_structure_type: + case dwarf::DW_TAG_typedef: + case dwarf::DW_TAG_unspecified_type: + if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_name)) + return true; + return false; + case dwarf::DW_TAG_namespace: + // According to DWARF5 spec namespaces without DW_AT_name needs to have + // "(anonymous namespace)" + if (!Die.findAttribute(dwarf::Attribute::DW_AT_name)) + NameToUse = "(anonymous namespace)"; + return true; + case dwarf::DW_TAG_inlined_subroutine: + case dwarf::DW_TAG_label: + case dwarf::DW_TAG_subprogram: + if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) || + Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) || + Die.findAttribute(dwarf::Attribute::DW_AT_ranges) || + Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc)) + return true; + return false; + case dwarf::DW_TAG_variable: + return TagsOnly || shouldIncludeVariable(Unit, Die); + default: + break; + } + return false; +} + +bool DWARF5AcceleratorTable::canGenerateEntryWithCrossCUReference( + const DWARFUnit &Unit, const DIE &Die, + const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec) { + if (!isCreated()) + return false; + std::string NameToUse = ""; + if (!canProcess(Unit, Die, NameToUse, true)) + return false; + return (AttrSpec.Attr == dwarf::Attribute::DW_AT_abstract_origin || + AttrSpec.Attr == dwarf::Attribute::DW_AT_specification) && + AttrSpec.Form == dwarf::DW_FORM_ref_addr; +} /// Returns name offset in String Offset section. static uint64_t getNameOffset(BinaryContext &BC, DWARFUnit &Unit, const uint64_t Index) { @@ -175,41 +224,6 @@ DWARF5AcceleratorTable::addAccelTableEntry( if (Unit.getVersion() < 5 || !NeedToCreate) return std::nullopt; std::string NameToUse = ""; - auto canProcess = [&](const DIE &Die) -> bool { - switch (Die.getTag()) { - case dwarf::DW_TAG_base_type: - case dwarf::DW_TAG_class_type: - case dwarf::DW_TAG_enumeration_type: - case dwarf::DW_TAG_imported_declaration: - case dwarf::DW_TAG_pointer_type: - case dwarf::DW_TAG_structure_type: - case dwarf::DW_TAG_typedef: - case dwarf::DW_TAG_unspecified_type: - if (Die.findAttribute(dwarf::Attribute::DW_AT_name)) - return true; - return false; - case dwarf::DW_TAG_namespace: - // According to DWARF5 spec namespaces without DW_AT_name needs to have - // "(anonymous namespace)" - if (!Die.findAttribute(dwarf::Attribute::DW_AT_name)) - NameToUse = "(anonymous namespace)"; - return true; - case dwarf::DW_TAG_inlined_subroutine: - case dwarf::DW_TAG_label: - case dwarf::DW_TAG_subprogram: - if (Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) || - Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) || - Die.findAttribute(dwarf::Attribute::DW_AT_ranges) || - Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc)) - return true; - return false; - case dwarf::DW_TAG_variable: - return shouldIncludeVariable(Unit, Die); - default: - break; - } - return false; - }; auto getUnitID = [&](const DWARFUnit &Unit, bool &IsTU, uint32_t &DieTag) -> uint32_t { @@ -223,7 +237,7 @@ DWARF5AcceleratorTable::addAccelTableEntry( return CUList.size() - 1; }; - if (!canProcess(Die)) + if (!canProcess(Unit, Die, NameToUse, false)) return std::nullopt; // Addes a Unit to either CU, LocalTU or ForeignTU list the first time we @@ -318,10 +332,24 @@ DWARF5AcceleratorTable::addAccelTableEntry( const DIEValue Value = Die.findAttribute(Attr); if (!Value) return std::nullopt; - const DIEEntry &DIEENtry = Value.getDIEEntry(); - DIE &EntryDie = DIEENtry.getEntry(); - addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_linkage_name)); - return addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_name)); + const DIE *EntryDie = nullptr; + if (Value.getForm() == dwarf::DW_FORM_ref_addr) { + auto Iter = CrossCUDies.find(Value.getDIEInteger().getValue()); + if (Iter == CrossCUDies.end()) { + BC.errs() << "BOLT-WARNING: [internal-dwarf-warning]: Could not find " + "referenced DIE in CrossCUDies for " + << Twine::utohexstr(Value.getDIEInteger().getValue()) + << ".\n"; + return std::nullopt; + } + EntryDie = Iter->second; + } else { + const DIEEntry &DIEENtry = Value.getDIEEntry(); + EntryDie = &DIEENtry.getEntry(); + } + + addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_linkage_name)); + return addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_name)); }; if (std::optional Entry = @@ -332,7 +360,6 @@ DWARF5AcceleratorTable::addAccelTableEntry( return *Entry; return addEntry(Die.findAttribute(dwarf::Attribute::DW_AT_name)); - ; } /// Algorithm from llvm implementation. diff --git a/bolt/test/X86/dwarf5-debug-names-cross-cu.s b/bolt/test/X86/dwarf5-debug-names-cross-cu.s new file mode 100644 index 000000000000..73c50d6d41db --- /dev/null +++ b/bolt/test/X86/dwarf5-debug-names-cross-cu.s @@ -0,0 +1,712 @@ + +# REQUIRES: system-linux + +# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o +# RUN: %clang %cflags -dwarf-5 %tmain.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections +# RUN: llvm-dwarfdump --debug-info -r 0 --debug-names %t.bolt > %t.txt +# RUN: cat %t.txt | FileCheck --check-prefix=CHECK %s + +## This test checks that BOLT generates Entries for DW_AT_abstract_origin when it has cross cu reference. + +# CHECK: [[OFFSET1:0x[0-9a-f]*]]: Compile Unit +# CHECK: [[OFFSET2:0x[0-9a-f]*]]: Compile Unit +# CHECK: Name Index @ 0x0 { +# CHECK-NEXT: Header { +# CHECK-NEXT: Length: 0xD2 +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: Version: 5 +# CHECK-NEXT: CU count: 2 +# CHECK-NEXT: Local TU count: 0 +# CHECK-NEXT: Foreign TU count: 0 +# CHECK-NEXT: Bucket count: 5 +# CHECK-NEXT: Name count: 5 +# CHECK-NEXT: Abbreviations table size: 0x1F +# CHECK-NEXT: Augmentation: 'BOLT' +# CHECK-NEXT: } +# CHECK-NEXT: Compilation Unit offsets [ +# CHECK-NEXT: CU[0]: [[OFFSET1]] +# CHECK-NEXT: CU[1]: [[OFFSET2]] +# CHECK-NEXT: ] +# CHECK-NEXT: Abbreviations [ +# CHECK-NEXT: Abbreviation [[ABBREV1:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_flag_present +# CHECK-NEXT: } +# CHECK-NEXT: Abbreviation [[ABBREV2:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_ref4 +# CHECK-NEXT: } +# CHECK-NEXT: Abbreviation [[ABBREV3:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_flag_present +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 0 [ +# CHECK-NEXT: EMPTY +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 1 [ +# CHECK-NEXT: Name 1 { +# CHECK-NEXT: Hash: 0x7C9A7F6A +# CHECK-NEXT: String: {{.+}} "main" +# CHECK-NEXT: Entry @ [[ENTRY:0x[0-9a-f]*]] { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000024 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: Name 2 { +# CHECK-NEXT: Hash: 0xB5063CFE +# CHECK-NEXT: String: {{.+}} "_Z3fooi" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x0000003a +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV2]] +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000054 +# CHECK-NEXT: DW_IDX_parent: Entry @ [[ENTRY]] +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 2 [ +# CHECK-NEXT: EMPTY +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 3 [ +# CHECK-NEXT: Name 3 { +# CHECK-NEXT: Hash: 0xB888030 +# CHECK-NEXT: String: {{.+}} "int" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV3]] +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000036 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 4 [ +# CHECK-NEXT: Name 4 { +# CHECK-NEXT: Hash: 0xB887389 +# CHECK-NEXT: String: {{.+}} "foo" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x0000003a +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: Entry @ 0xc4 { +# CHECK-NEXT: Abbrev: [[ABBREV2]] +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000054 +# CHECK-NEXT: DW_IDX_parent: Entry @ [[ENTRY]] +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: Name 5 { +# CHECK-NEXT: Hash: 0x7C952063 +# CHECK-NEXT: String: {{.+}} "char" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV3]] +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000075 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: } + +## clang++ -g2 -gpubnames -S -emit-llvm main.cpp -o main.ll +## clang++ -g2 -gpubnames -S -emit-llvm helper.cpp -o helper.ll +## llvm-link main.ll helper.ll -o combined.ll +## clang++ -g2 -gpubnames combined.ll -emit-llvm -S -o combined.opt.ll +## llc -dwarf-version=5 -filetype=asm -mtriple x86_64-unknown-linux combined.opt.ll -o combined.s +## main.cpp +## extern int foo(int); +## int main(int argc, char* argv[]) { +## int i = 0; +## [[clang::always_inline]] i = foo(argc); +## return i; +## } +## helper.cpp +## int foo(int i) { +## return i ++; +## } + + .text + .file "llvm-link" + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .file 1 "/home" "main.cpp" md5 0x24fb0b4c3900e91fece1ac87ed73ff3b + .loc 1 2 0 # main.cpp:2:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl $0, -16(%rbp) + movl %edi, -12(%rbp) + movq %rsi, -24(%rbp) +.Ltmp0: + .loc 1 3 7 prologue_end # main.cpp:3:7 + movl $0, -4(%rbp) + .loc 1 4 36 # main.cpp:4:36 + movl -12(%rbp), %eax + movl %eax, -8(%rbp) +.Ltmp1: + .file 2 "/home" "helper.cpp" md5 0x7d4429e24d8c74d7ee22c1889ad46d6b + .loc 2 2 12 # helper.cpp:2:12 + movl -8(%rbp), %eax + movl %eax, %ecx + addl $1, %ecx + movl %ecx, -8(%rbp) +.Ltmp2: + .loc 1 4 30 # main.cpp:4:30 + movl %eax, -4(%rbp) + .loc 1 5 10 # main.cpp:5:10 + movl -4(%rbp), %eax + .loc 1 5 3 epilogue_begin is_stmt 0 # main.cpp:5:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp3: +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + # -- End function + .globl _Z3fooi # -- Begin function _Z3fooi + .p2align 4, 0x90 + .type _Z3fooi,@function +_Z3fooi: # @_Z3fooi +.Lfunc_begin1: + .loc 2 1 0 is_stmt 1 # helper.cpp:1:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl %edi, -4(%rbp) +.Ltmp4: + .loc 2 2 12 prologue_end # helper.cpp:2:12 + movl -4(%rbp), %eax + movl %eax, %ecx + addl $1, %ecx + movl %ecx, -4(%rbp) + .loc 2 2 3 epilogue_begin is_stmt 0 # helper.cpp:2:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp5: +.Lfunc_end1: + .size _Z3fooi, .Lfunc_end1-_Z3fooi + .cfi_endproc + # -- End function + .section .debug_abbrev,"",@progbits + .byte 1 # Abbreviation Code + .byte 17 # DW_TAG_compile_unit + .byte 1 # DW_CHILDREN_yes + .byte 37 # DW_AT_producer + .byte 37 # DW_FORM_strx1 + .byte 19 # DW_AT_language + .byte 5 # DW_FORM_data2 + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 114 # DW_AT_str_offsets_base + .byte 23 # DW_FORM_sec_offset + .byte 16 # DW_AT_stmt_list + .byte 23 # DW_FORM_sec_offset + .byte 27 # DW_AT_comp_dir + .byte 37 # DW_FORM_strx1 + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 115 # DW_AT_addr_base + .byte 23 # DW_FORM_sec_offset + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 2 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 3 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 4 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 5 # Abbreviation Code + .byte 52 # DW_TAG_variable + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 6 # Abbreviation Code + .byte 29 # DW_TAG_inlined_subroutine + .byte 1 # DW_CHILDREN_yes + .byte 49 # DW_AT_abstract_origin + .byte 16 # DW_FORM_ref_addr + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 88 # DW_AT_call_file + .byte 11 # DW_FORM_data1 + .byte 89 # DW_AT_call_line + .byte 11 # DW_FORM_data1 + .byte 87 # DW_AT_call_column + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 7 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 8 # Abbreviation Code + .byte 15 # DW_TAG_pointer_type + .byte 0 # DW_CHILDREN_no + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 9 # Abbreviation Code + .byte 36 # DW_TAG_base_type + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 62 # DW_AT_encoding + .byte 11 # DW_FORM_data1 + .byte 11 # DW_AT_byte_size + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 10 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 110 # DW_AT_linkage_name + .byte 37 # DW_FORM_strx1 + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 32 # DW_AT_inline + .byte 33 # DW_FORM_implicit_const + .byte 1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 11 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 12 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 13 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 0 # EOM(3) + .section .debug_info,"",@progbits +.Lcu_begin0: + .long .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit +.Ldebug_info_start0: + .short 5 # DWARF version number + .byte 1 # DWARF Unit Type + .byte 8 # Address Size (in bytes) + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 1 # Abbrev [1] 0xc:0x6d DW_TAG_compile_unit + .byte 0 # DW_AT_producer + .short 33 # DW_AT_language + .byte 1 # DW_AT_name + .long .Lstr_offsets_base0 # DW_AT_str_offsets_base + .long .Lline_table_start0 # DW_AT_stmt_list + .byte 2 # DW_AT_comp_dir + .byte 0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .long .Laddr_table_base0 # DW_AT_addr_base + .byte 2 # Abbrev [2] 0x23:0x47 DW_TAG_subprogram + .byte 0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .byte 8 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + # DW_AT_external + .byte 3 # Abbrev [3] 0x32:0xb DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 116 + .byte 9 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + .byte 4 # Abbrev [4] 0x3d:0xb DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 104 + .byte 10 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long 106 # DW_AT_type + .byte 5 # Abbrev [5] 0x48:0xb DW_TAG_variable + .byte 2 # DW_AT_location + .byte 145 + .byte 124 + .byte 7 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 3 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + .byte 6 # Abbrev [6] 0x53:0x16 DW_TAG_inlined_subroutine + .long .debug_info+156 # DW_AT_abstract_origin + .byte 1 # DW_AT_low_pc + .long .Ltmp2-.Ltmp1 # DW_AT_high_pc + .byte 1 # DW_AT_call_file + .byte 4 # DW_AT_call_line + .byte 32 # DW_AT_call_column + .byte 7 # Abbrev [7] 0x60:0x8 DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 120 + .long .debug_info+165 # DW_AT_abstract_origin + .byte 0 # End Of Children Mark + .byte 0 # End Of Children Mark + .byte 8 # Abbrev [8] 0x6a:0x5 DW_TAG_pointer_type + .long 111 # DW_AT_type + .byte 8 # Abbrev [8] 0x6f:0x5 DW_TAG_pointer_type + .long 116 # DW_AT_type + .byte 9 # Abbrev [9] 0x74:0x4 DW_TAG_base_type + .byte 11 # DW_AT_name + .byte 6 # DW_AT_encoding + .byte 1 # DW_AT_byte_size + .byte 0 # End Of Children Mark +.Ldebug_info_end0: +.Lcu_begin1: + .long .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit +.Ldebug_info_start1: + .short 5 # DWARF version number + .byte 1 # DWARF Unit Type + .byte 8 # Address Size (in bytes) + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 1 # Abbrev [1] 0xc:0x43 DW_TAG_compile_unit + .byte 0 # DW_AT_producer + .short 33 # DW_AT_language + .byte 3 # DW_AT_name + .long .Lstr_offsets_base0 # DW_AT_str_offsets_base + .long .Lline_table_start0 # DW_AT_stmt_list + .byte 2 # DW_AT_comp_dir + .byte 2 # DW_AT_low_pc + .long .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc + .long .Laddr_table_base0 # DW_AT_addr_base + .byte 10 # Abbrev [10] 0x23:0x12 DW_TAG_subprogram + .byte 4 # DW_AT_linkage_name + .byte 5 # DW_AT_name + .byte 2 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 53 # DW_AT_type + # DW_AT_external + # DW_AT_inline + .byte 11 # Abbrev [11] 0x2c:0x8 DW_TAG_formal_parameter + .byte 7 # DW_AT_name + .byte 2 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 53 # DW_AT_type + .byte 0 # End Of Children Mark + .byte 9 # Abbrev [9] 0x35:0x4 DW_TAG_base_type + .byte 6 # DW_AT_name + .byte 5 # DW_AT_encoding + .byte 4 # DW_AT_byte_size + .byte 12 # Abbrev [12] 0x39:0x15 DW_TAG_subprogram + .byte 2 # DW_AT_low_pc + .long .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .long 35 # DW_AT_abstract_origin + .byte 13 # Abbrev [13] 0x45:0x8 DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 124 + .long 44 # DW_AT_abstract_origin + .byte 0 # End Of Children Mark + .byte 0 # End Of Children Mark +.Ldebug_info_end1: + .section .debug_str_offsets,"",@progbits + .long 52 # Length of String Offsets Set + .short 5 + .short 0 +.Lstr_offsets_base0: + .section .debug_str,"MS",@progbits,1 +.Linfo_string0: + .asciz "clang version 19.0.0git" # string offset=0 +.Linfo_string1: + .asciz "main.cpp" # string offset=24 +.Linfo_string2: + .asciz "/home/ayermolo/local/tasks/T182867349" # string offset=33 +.Linfo_string3: + .asciz "helper.cpp" # string offset=71 +.Linfo_string4: + .asciz "_Z3fooi" # string offset=82 +.Linfo_string5: + .asciz "foo" # string offset=90 +.Linfo_string6: + .asciz "int" # string offset=94 +.Linfo_string7: + .asciz "i" # string offset=98 +.Linfo_string8: + .asciz "main" # string offset=100 +.Linfo_string9: + .asciz "argc" # string offset=105 +.Linfo_string10: + .asciz "argv" # string offset=110 +.Linfo_string11: + .asciz "char" # string offset=115 + .section .debug_str_offsets,"",@progbits + .long .Linfo_string0 + .long .Linfo_string1 + .long .Linfo_string2 + .long .Linfo_string3 + .long .Linfo_string4 + .long .Linfo_string5 + .long .Linfo_string6 + .long .Linfo_string7 + .long .Linfo_string8 + .long .Linfo_string9 + .long .Linfo_string10 + .long .Linfo_string11 + .section .debug_addr,"",@progbits + .long .Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution +.Ldebug_addr_start0: + .short 5 # DWARF version number + .byte 8 # Address size + .byte 0 # Segment selector size +.Laddr_table_base0: + .quad .Lfunc_begin0 + .quad .Ltmp1 + .quad .Lfunc_begin1 +.Ldebug_addr_end0: + .section .debug_names,"",@progbits + .long .Lnames_end0-.Lnames_start0 # Header: unit length +.Lnames_start0: + .short 5 # Header: version + .short 0 # Header: padding + .long 2 # Header: compilation unit count + .long 0 # Header: local type unit count + .long 0 # Header: foreign type unit count + .long 5 # Header: bucket count + .long 5 # Header: name count + .long .Lnames_abbrev_end0-.Lnames_abbrev_start0 # Header: abbreviation table size + .long 8 # Header: augmentation string size + .ascii "LLVM0700" # Header: augmentation string + .long .Lcu_begin0 # Compilation unit 0 + .long .Lcu_begin1 # Compilation unit 1 + .long 0 # Bucket 0 + .long 1 # Bucket 1 + .long 0 # Bucket 2 + .long 3 # Bucket 3 + .long 4 # Bucket 4 + .long 2090499946 # Hash in Bucket 1 + .long -1257882370 # Hash in Bucket 1 + .long 193495088 # Hash in Bucket 3 + .long 193491849 # Hash in Bucket 4 + .long 2090147939 # Hash in Bucket 4 + .long .Linfo_string8 # String in Bucket 1: main + .long .Linfo_string4 # String in Bucket 1: _Z3fooi + .long .Linfo_string6 # String in Bucket 3: int + .long .Linfo_string5 # String in Bucket 4: foo + .long .Linfo_string11 # String in Bucket 4: char + .long .Lnames1-.Lnames_entries0 # Offset in Bucket 1 + .long .Lnames3-.Lnames_entries0 # Offset in Bucket 1 + .long .Lnames0-.Lnames_entries0 # Offset in Bucket 3 + .long .Lnames2-.Lnames_entries0 # Offset in Bucket 4 + .long .Lnames4-.Lnames_entries0 # Offset in Bucket 4 +.Lnames_abbrev_start0: + .byte 1 # Abbrev code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 25 # DW_FORM_flag_present + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 2 # Abbrev code + .byte 29 # DW_TAG_inlined_subroutine + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 19 # DW_FORM_ref4 + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 3 # Abbrev code + .byte 36 # DW_TAG_base_type + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 25 # DW_FORM_flag_present + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 0 # End of abbrev list +.Lnames_abbrev_end0: +.Lnames_entries0: +.Lnames1: +.L3: + .byte 1 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 35 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: main +.Lnames3: +.L0: + .byte 1 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 57 # DW_IDX_die_offset +.L2: # DW_IDX_parent + .byte 2 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 83 # DW_IDX_die_offset + .long .L3-.Lnames_entries0 # DW_IDX_parent + .byte 0 # End of list: _Z3fooi +.Lnames0: +.L4: + .byte 3 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 53 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: int +.Lnames2: + .byte 1 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 57 # DW_IDX_die_offset + .byte 2 # DW_IDX_parent + # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 83 # DW_IDX_die_offset + .long .L3-.Lnames_entries0 # DW_IDX_parent + .byte 0 # End of list: foo +.Lnames4: +.L1: + .byte 3 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 116 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: char + .p2align 2, 0x0 +.Lnames_end0: + .ident "clang version 19.0.0git" + .ident "clang version 19.0.0git" + .section ".note.GNU-stack","",@progbits + .section .debug_line,"",@progbits +.Lline_table_start0: -- GitLab From dcbddc25250158469c5635ad2ae4095faef53dfd Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 16:13:58 -0500 Subject: [PATCH 210/933] [Libomptarget] Unify and simplify plugin CMake (#86191) Summary: This patch reworks the CMake handling for building plugins. All this does is pull a lot of shared and common logic into a single helper function. This also simplifies the OMPT libraries from being built separately instead of just added. --- .../plugins-nextgen/CMakeLists.txt | 59 ++++++++++++++ .../plugins-nextgen/amdgpu/CMakeLists.txt | 80 +++---------------- .../plugins-nextgen/common/CMakeLists.txt | 54 ++----------- .../common/OMPT/CMakeLists.txt | 70 ---------------- .../plugins-nextgen/cuda/CMakeLists.txt | 40 ++-------- .../plugins-nextgen/host/CMakeLists.txt | 29 +------ 6 files changed, 87 insertions(+), 245 deletions(-) delete mode 100644 openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index 75540f055844..8e8b040fd562 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -10,7 +10,66 @@ # ##===----------------------------------------------------------------------===## +# Common interface to handle creating a plugin library. +set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) +function(add_target_library target_name lib_name) + llvm_map_components_to_libnames(llvm_libs + ${LLVM_TARGETS_TO_BUILD} + AggressiveInstCombine + Analysis + BinaryFormat + BitReader + BitWriter + CodeGen + Core + Extensions + InstCombine + Instrumentation + IPO + IRReader + Linker + MC + Object + Passes + Remarks + ScalarOpts + Support + Target + TargetParser + TransformUtils + Vectorize + ) + + add_llvm_library(${target_name} SHARED + NO_INSTALL_RPATH + BUILDTREE_ONLY + ) + + llvm_update_compile_flags(${target_name}) + target_link_libraries(${target_name} PUBLIC + PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) + + target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) + target_compile_definitions(${target_name} PRIVATE + DEBUG_PREFIX="TARGET ${lib_name} RTL") + + if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by + # the dynamic linker at runtime. Therefore, allow the symbol to be undefined + # when creating a shared library. + target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined") + else() + target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs") + endif() + + if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) + target_link_libraries(${target_name} PRIVATE + "-Wl,--version-script=${common_dir}/../exports") + endif() + set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected) +endfunction() + add_subdirectory(amdgpu) add_subdirectory(cuda) add_subdirectory(host) diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt index 8fbfe4d9b13f..40df77102c78 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt @@ -27,76 +27,23 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE return() endif() -################################################################################ -# Define the suffix for the runtime messaging dumps. -add_definitions(-DTARGET_NAME=AMDGPU) - -# Define debug prefix. TODO: This should be automatized in the Debug.h but it -# requires changing the original plugins. -add_definitions(-DDEBUG_PREFIX="TARGET AMDGPU RTL") +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.amdgpu AMDGPU) -set(LIBOMPTARGET_DLOPEN_LIBHSA OFF) -option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" ${LIBOMPTARGET_DLOPEN_LIBHSA}) - -if (${hsa-runtime64_FOUND} AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA) - libomptarget_say("Building AMDGPU NextGen plugin linked against libhsa") - set(LIBOMPTARGET_EXTRA_SOURCE) - set(LIBOMPTARGET_DEP_LIBRARIES hsa-runtime64::hsa-runtime64) -else() - libomptarget_say("Building AMDGPU NextGen plugin for dlopened libhsa") - include_directories(dynamic_hsa) - set(LIBOMPTARGET_EXTRA_SOURCE dynamic_hsa/hsa.cpp) - set(LIBOMPTARGET_DEP_LIBRARIES) -endif() +target_sources(omptarget.rtl.amdgpu PRIVATE src/rtl.cpp) +target_include_directories(omptarget.rtl.amdgpu PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/utils) -if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by - # the dynamic linker at runtime. Therefore, allow the symbol to be undefined - # when creating a shared library. - set(LDFLAGS_UNDEFINED "-Wl,--allow-shlib-undefined") +option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" OFF) +if(hsa-runtime64_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA) + libomptarget_say("Building AMDGPU plugin linked against libhsa") + target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64) else() - set(LDFLAGS_UNDEFINED "-Wl,-z,defs") + libomptarget_say("Building AMDGPU plugin for dlopened libhsa") + target_include_directories(omptarget.rtl.amdgpu PRIVATE dynamic_hsa) + target_sources(omptarget.rtl.amdgpu PRIVATE dynamic_hsa/hsa.cpp) endif() -add_llvm_library(omptarget.rtl.amdgpu SHARED - src/rtl.cpp - ${LIBOMPTARGET_EXTRA_SOURCE} - - ADDITIONAL_HEADER_DIRS - ${LIBOMPTARGET_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/utils - - LINK_COMPONENTS - Support - Object - - LINK_LIBS - PRIVATE - PluginCommon - ${LIBOMPTARGET_DEP_LIBRARIES} - ${OPENMP_PTHREAD_LIB} - ${LDFLAGS_UNDEFINED} - - NO_INSTALL_RPATH - BUILDTREE_ONLY -) - -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(omptarget.rtl.amdgpu PRIVATE OMPT) -endif() - -if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.amdgpu PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") -endif() - -target_include_directories( - omptarget.rtl.amdgpu - PRIVATE - ${LIBOMPTARGET_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/utils -) - # Configure testing for the AMDGPU plugin. We will build tests if we could a # functional AMD GPU on the system, or if manually specifies by the user. option(LIBOMPTARGET_FORCE_AMDGPU_TESTS "Build AMDGPU libomptarget tests" OFF) @@ -114,5 +61,4 @@ endif() # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") set_target_properties(omptarget.rtl.amdgpu PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - CXX_VISIBILITY_PRESET protected) + INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 085d44307165..0420d0e6f1f8 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -26,45 +26,6 @@ foreach(Target ${TargetsSupported}) target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${Target}") endforeach() -# This is required when using LLVM libraries. -llvm_update_compile_flags(PluginCommon) - -if (LLVM_LINK_LLVM_DYLIB) - set(llvm_libs LLVM) -else() - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) -endif() - -target_link_libraries(PluginCommon - PUBLIC - ${llvm_libs} -) - # Include the RPC server from the `libc` project if availible. if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT}) target_link_libraries(PluginCommon PRIVATE llvmlibc_rpc_server) @@ -82,8 +43,10 @@ elseif(${LIBOMPTARGET_GPU_LIBC_SUPPORT}) endif() endif() -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(PluginCommon PUBLIC OMPT) +# If we have OMPT enabled include it in the list of sources. +if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) + target_sources(PluginCommon PRIVATE OMPT/OmptCallback.cpp) + target_include_directories(PluginCommon PRIVATE OMPT) endif() # Define the TARGET_NAME and DEBUG_PREFIX. @@ -95,16 +58,11 @@ target_compile_definitions(PluginCommon PRIVATE target_compile_options(PluginCommon PUBLIC ${offload_compile_flags}) target_link_options(PluginCommon PUBLIC ${offload_link_flags}) -target_include_directories(PluginCommon - PRIVATE - ${LIBOMPTARGET_INCLUDE_DIR} - PUBLIC +target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_INCLUDE_DIR} ) set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_VISIBILITY_PRESET protected) - -add_subdirectory(OMPT) - diff --git a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt deleted file mode 100644 index be4c743665b3..000000000000 --- a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt +++ /dev/null @@ -1,70 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Aggregation of parts which can be used by OpenMP tools -# -##===----------------------------------------------------------------------===## - -# NOTE: Don't try to build `OMPT` using `add_llvm_library` because we -# don't want to export `OMPT` while `add_llvm_library` requires that. -add_library(OMPT OBJECT - OmptCallback.cpp) - -# This is required when using LLVM libraries. -llvm_update_compile_flags(OMPT) - -if (LLVM_LINK_LLVM_DYLIB) - set(llvm_libs LLVM) -else() - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) -endif() - -target_link_libraries(OMPT - PUBLIC - ${llvm_libs} -) - -# Define the TARGET_NAME and DEBUG_PREFIX. -target_compile_definitions(OMPT PRIVATE - TARGET_NAME="OMPT" - DEBUG_PREFIX="OMPT" -) - -target_include_directories(OMPT - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${LIBOMPTARGET_INCLUDE_DIR} -) - -set_target_properties(OMPT PROPERTIES - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET protected) diff --git a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt index 2bfb47168a7f..b3530462aa19 100644 --- a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt @@ -23,34 +23,12 @@ endif() libomptarget_say("Building CUDA NextGen offloading plugin.") -set(LIBOMPTARGET_DLOPEN_LIBCUDA OFF) -option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" ${LIBOMPTARGET_DLOPEN_LIBCUDA}) - -add_llvm_library(omptarget.rtl.cuda SHARED - src/rtl.cpp - - LINK_COMPONENTS - Support - Object - - LINK_LIBS PRIVATE - PluginCommon - ${OPENMP_PTHREAD_LIB} - - NO_INSTALL_RPATH - BUILDTREE_ONLY -) - -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(omptarget.rtl.cuda PRIVATE OMPT) -endif() - -if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.cuda PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports,-z,defs") -endif() +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.cuda CUDA) +target_sources(omptarget.rtl.cuda PRIVATE src/rtl.cpp) +option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" OFF) if(LIBOMPTARGET_DEP_CUDA_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA) libomptarget_say("Building CUDA plugin linked against libcuda") target_link_libraries(omptarget.rtl.cuda PRIVATE CUDA::cuda_driver) @@ -60,13 +38,6 @@ else() target_sources(omptarget.rtl.cuda PRIVATE dynamic_cuda/cuda.cpp) endif() -# Define debug prefix. TODO: This should be automatized in the Debug.h but it -# requires changing the original plugins. -target_compile_definitions(omptarget.rtl.cuda PRIVATE TARGET_NAME="CUDA") -target_compile_definitions(omptarget.rtl.cuda PRIVATE DEBUG_PREFIX="TARGET CUDA RTL") - -target_include_directories(omptarget.rtl.cuda PRIVATE ${LIBOMPTARGET_INCLUDE_DIR}) - # Configure testing for the CUDA plugin. We will build tests if we could a # functional NVIDIA GPU on the system, or if manually specifies by the user. option(LIBOMPTARGET_FORCE_NVIDIA_TESTS "Build NVIDIA libomptarget tests" OFF) @@ -84,5 +55,4 @@ endif() # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") set_target_properties(omptarget.rtl.cuda PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - CXX_VISIBILITY_PRESET protected) + INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index 58a79898ff80..d30680e10431 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -2,11 +2,6 @@ if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") return() endif() - # build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") - # build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") - # build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") - # build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") - # build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") set(supported_targets x86_64 aarch64 ppc64 ppc64le s390x) if(NOT ${CMAKE_SYSTEM_PROCESSOR} IN_LIST supported_targets) libomptarget_say("Not building ${machine} NextGen offloading plugin") @@ -18,16 +13,10 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$") set(machine ppc64) endif() -add_llvm_library(omptarget.rtl.${machine} SHARED - src/rtl.cpp - ADDITIONAL_HEADER_DIRS - ${LIBOMPTARGET_INCLUDE_DIR} - LINK_LIBS PRIVATE - PluginCommon - ${OPENMP_PTHREAD_LIB} - NO_INSTALL_RPATH - BUILDTREE_ONLY -) +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.${machine} ${machine}) + +target_sources(omptarget.rtl.${machine} PRIVATE src/rtl.cpp) if(LIBOMPTARGET_DEP_LIBFFI_FOUND) libomptarget_say("Building ${machine} plugin linked with libffi") @@ -42,10 +31,6 @@ else() target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) endif() -if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) - target_link_libraries(omptarget.rtl.${machine} PRIVATE OMPT) -endif() - if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) target_link_libraries(omptarget.rtl.${machine} PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") @@ -70,12 +55,6 @@ else() libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.") endif() -# Define macro to be used as prefix of the runtime messages for this target. -target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_NAME=${machine}) -# TODO: This should be automatized in Debug.h. -target_compile_definitions(omptarget.rtl.${machine} PRIVATE - DEBUG_PREFIX="TARGET ${machine} RTL") - # Define the target specific triples and ELF machine values. if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$" OR CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$") -- GitLab From 215f105ca5d0b42d00bbbc315605b222d63be63a Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 22 Mar 2024 14:14:43 -0700 Subject: [PATCH 211/933] [MTE] Fix test (#85875) llc runs the stack tagging instrumentation, so if we run opt before, we double instrument --- llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll index 81349620fb77..5d1c91e434dc 100644 --- a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll +++ b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll @@ -1,17 +1,15 @@ ; Test that storage for allocas with disjoint lifetimes is reused with stack ; tagging. -; RUN: opt -S -aarch64-stack-tagging -stack-tagging-use-stack-safety=0 %s -o - | \ -; RUN: llc --mattr=+mte -no-stack-coloring=false -o - | \ +; RUN: llc --mattr=+mte -no-stack-coloring=false -stack-tagging-use-stack-safety=0 -o - %s | \ ; RUN: FileCheck %s --check-prefix=COLOR -; RUN: opt -S -aarch64-stack-tagging %s -stack-tagging-use-stack-safety=0 -o - | \ -; RUN: llc --mattr=+mte -no-stack-coloring=true -o - | \ +; RUN: llc --mattr=+mte -no-stack-coloring=true -stack-tagging-use-stack-safety=0 -o - %s | \ ; RUN: FileCheck %s --check-prefix=NOCOLOR target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64" -; COLOR: sub sp, sp, #208 +; COLOR: sub sp, sp, #192 ; NOCOLOR: sub sp, sp, #336 define i32 @myCall_w2(i32 %in) sanitize_memtag { -- GitLab From 14be4930c10bcc6f6c0096097350cb3cafff9661 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 14:17:51 -0700 Subject: [PATCH 212/933] [bazel] Make compiler-rt analyze on macOS (#86001) Previously the select above would fail for non-linux platforms if you did a `bazel build @llvm-project//...`, now this target specifies that it's only supported on the linux platform through bazel's `target_compatible_with` feature. This makes all targets in the tree be ignored when building on incompatible platforms (and fail if built directly) --- utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel index 9bdd454e1e36..577e6c033b4e 100644 --- a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel @@ -19,6 +19,10 @@ cc_library( ], # Will raise error unless supported platforms. }), + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) WIN32_ONLY_FILES = [ -- GitLab From 85af772f3b4067fce703b33cee0e2cdafc74a6d6 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 16:38:10 -0500 Subject: [PATCH 213/933] [Libomptarget][FIX] Fix unintentinally used PUBLIC interface Summary: This was supposed to be private and caused some issues with certain configs. --- openmp/libomptarget/plugins-nextgen/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index 8e8b040fd562..c19fd80592d6 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -47,7 +47,7 @@ function(add_target_library target_name lib_name) ) llvm_update_compile_flags(${target_name}) - target_link_libraries(${target_name} PUBLIC + target_link_libraries(${target_name} PRIVATE PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) -- GitLab From 5d0d9eb52dbb3bcf6f500c7b18d58c8bdf6659ce Mon Sep 17 00:00:00 2001 From: Ellis Hoag Date: Fri, 22 Mar 2024 14:47:31 -0700 Subject: [PATCH 214/933] [NFC][BP] Remove unused parameter from function (#86333) Remove the unused parameter `RecDepth` from `runIterations()`. --- llvm/include/llvm/Support/BalancedPartitioning.h | 5 ++--- llvm/lib/Support/BalancedPartitioning.cpp | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/Support/BalancedPartitioning.h b/llvm/include/llvm/Support/BalancedPartitioning.h index 9738e742f7f1..539d157343fb 100644 --- a/llvm/include/llvm/Support/BalancedPartitioning.h +++ b/llvm/include/llvm/Support/BalancedPartitioning.h @@ -142,9 +142,8 @@ private: std::optional &TP) const; /// Run bisection iterations - void runIterations(const FunctionNodeRange Nodes, unsigned RecDepth, - unsigned LeftBucket, unsigned RightBucket, - std::mt19937 &RNG) const; + void runIterations(const FunctionNodeRange Nodes, unsigned LeftBucket, + unsigned RightBucket, std::mt19937 &RNG) const; /// Run a bisection iteration to improve the optimization goal /// \returns the total number of moved FunctionNodes diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp index f4254b50d26c..141f0034a23f 100644 --- a/llvm/lib/Support/BalancedPartitioning.cpp +++ b/llvm/lib/Support/BalancedPartitioning.cpp @@ -136,7 +136,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes, // Split into two and assign to the left and right buckets split(Nodes, LeftBucket); - runIterations(Nodes, RecDepth, LeftBucket, RightBucket, RNG); + runIterations(Nodes, LeftBucket, RightBucket, RNG); // Split nodes wrt the resulting buckets auto NodesMid = @@ -163,7 +163,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes, } void BalancedPartitioning::runIterations(const FunctionNodeRange Nodes, - unsigned RecDepth, unsigned LeftBucket, + unsigned LeftBucket, unsigned RightBucket, std::mt19937 &RNG) const { unsigned NumNodes = std::distance(Nodes.begin(), Nodes.end()); -- GitLab From 4652ec0e291ca4ba4ddef3fd59b202646e9a6694 Mon Sep 17 00:00:00 2001 From: Patrick O'Neill <102189596+patrick-rivos@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:52:27 -0700 Subject: [PATCH 215/933] [SLP] Delete vectorized users when tree contains an invalid cost (#86344) --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 +- .../RISCV/partial-vec-invalid-cost.ll | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 0f7afa2fc25c..f98d15c285a6 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15959,7 +15959,7 @@ public: LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for reduction\n"); if (!Cost.isValid()) - return nullptr; + break; if (Cost >= -SLPCostThreshold) { V.getORE()->emit([&]() { return OptimizationRemarkMissed( diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll new file mode 100644 index 000000000000..31f16801b7a6 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll @@ -0,0 +1,57 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s + +target triple = "riscv64-unknown-linux-gnu" + +define void @partial_vec_invalid_cost() #0 { +; CHECK-LABEL: define void @partial_vec_invalid_cost( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 0 +; CHECK-NEXT: [[LSHR_2:%.*]] = lshr i96 0, 0 +; CHECK-NEXT: [[TRUNC_I96_1:%.*]] = trunc i96 [[LSHR_1]] to i32 +; CHECK-NEXT: [[TRUNC_I96_2:%.*]] = trunc i96 [[LSHR_2]] to i32 +; CHECK-NEXT: [[TRUNC_I96_3:%.*]] = trunc i96 0 to i32 +; CHECK-NEXT: [[TRUNC_I96_4:%.*]] = trunc i96 0 to i32 +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer) +; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP0]], [[TRUNC_I96_2]] +; CHECK-NEXT: [[OP_RDX1:%.*]] = or i32 [[TRUNC_I96_1]], [[TRUNC_I96_3]] +; CHECK-NEXT: [[OP_RDX2:%.*]] = or i32 [[OP_RDX]], [[OP_RDX1]] +; CHECK-NEXT: [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TRUNC_I96_4]] +; CHECK-NEXT: [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96 +; CHECK-NEXT: store i96 [[STORE_THIS]], ptr null, align 16 +; CHECK-NEXT: ret void +; +entry: + + %lshr.1 = lshr i96 0, 0 ; These ops + %lshr.2 = lshr i96 0, 0 ; return an + %add.0 = add i96 0, 0 ; invalid + %add.1 = add i96 0, 0 ; vector cost. + + %trunc.i96.1 = trunc i96 %lshr.1 to i32 ; These ops + %trunc.i96.2 = trunc i96 %lshr.2 to i32 ; return an + %trunc.i96.3 = trunc i96 %add.0 to i32 ; invalid + %trunc.i96.4 = trunc i96 %add.1 to i32 ; vector cost. + + %or.0 = or i32 %trunc.i96.1, %trunc.i96.2 + %or.1 = or i32 %or.0, %trunc.i96.3 + %or.2 = or i32 %or.1, %trunc.i96.4 + + %zext.0 = zext i1 0 to i32 ; These + %zext.1 = zext i1 0 to i32 ; ops + %zext.2 = zext i1 0 to i32 ; are + %zext.3 = zext i1 0 to i32 ; vectorized + + %or.3 = or i32 %or.2, %zext.0 ; users + %or.4 = or i32 %or.3, %zext.1 ; of + %or.5 = or i32 %or.4, %zext.2 ; vectorized + %or.6 = or i32 %or.5, %zext.3 ; ops + + %store.this = zext i32 %or.6 to i96 + + store i96 %store.this, ptr null, align 16 + ret void +} + +attributes #0 = { "target-features"="+v" } -- GitLab From 913e29966bac5fec08998a1acc3e793f9b7bcc12 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:11:26 -0700 Subject: [PATCH 216/933] [NFC][tsan] Use the result of placement new (#86341) --- compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp index 77488f843285..06c34a2e4383 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp @@ -200,9 +200,8 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, } void ThreadContext::OnStarted(void *arg) { - thr = static_cast(arg); DPrintf("#%d: ThreadStart\n", tid); - new (thr) ThreadState(tid); + thr = new (arg) ThreadState(tid); if (common_flags()->detect_deadlocks) thr->dd_lt = ctx->dd->CreateLogicalThread(tid); thr->tctx = this; -- GitLab From 0ba678a53d3ef7d125f38720a59875035739dc9b Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:12:09 -0700 Subject: [PATCH 217/933] [tsan] Set `thr->is_inited` after SlotAttachAndLock (#86342) Almost NFC. This is symmetrical to `ThreadFinish`, which resets the slot after `is_inited`. --- compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp index 06c34a2e4383..5316a7862e44 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp @@ -160,6 +160,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, } Free(thr->tctx->sync); +#if !SANITIZER_GO + thr->is_inited = true; +#endif + uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; @@ -205,9 +209,6 @@ void ThreadContext::OnStarted(void *arg) { if (common_flags()->detect_deadlocks) thr->dd_lt = ctx->dd->CreateLogicalThread(tid); thr->tctx = this; -#if !SANITIZER_GO - thr->is_inited = true; -#endif } void ThreadFinish(ThreadState *thr) { -- GitLab From 362d26366d0175f01ffb6085eb747a6e40f01147 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:24:44 -0700 Subject: [PATCH 218/933] [tsan] Process SIGPROF as sync signal only if thread is alive (#86343) Otherwise it may crash too early. This is followup to #85188 --- compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 2bebe651b994..810ce69663d0 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -2169,8 +2169,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig, return false; #endif return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || - sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || - sig == SIGPROF; + sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS; } void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { @@ -2181,7 +2180,8 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { return; } // Don't mess with synchronous signals. - const bool sync = is_sync_signal(sctx, sig, info); + const bool sync = is_sync_signal(sctx, sig, info) || + (sig == SIGPROF && thr->is_inited && !thr->is_dead); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, -- GitLab From b1e97d60bd5b1d3f994345caa4012ea11c2a0f62 Mon Sep 17 00:00:00 2001 From: Yeoul Na Date: Sat, 23 Mar 2024 07:26:35 +0900 Subject: [PATCH 219/933] Unwrap CountAttributed for debug info (#86017) Fix crash caused by 3eb9ff30959a670559bcba03d149d4c51bf7c9c9 --- clang/lib/CodeGen/CGDebugInfo.cpp | 3 +++ .../test/CodeGen/attr-counted-by-debug-info.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 clang/test/CodeGen/attr-counted-by-debug-info.c diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 07ecaa81c47d..7453ed14aef4 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3463,6 +3463,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { case Type::BTFTagAttributed: T = cast(T)->getWrappedType(); break; + case Type::CountAttributed: + T = cast(T)->desugar(); + break; case Type::Elaborated: T = cast(T)->getNamedType(); break; diff --git a/clang/test/CodeGen/attr-counted-by-debug-info.c b/clang/test/CodeGen/attr-counted-by-debug-info.c new file mode 100644 index 000000000000..a6c2b1382b79 --- /dev/null +++ b/clang/test/CodeGen/attr-counted-by-debug-info.c @@ -0,0 +1,18 @@ +// RUN: %clang -emit-llvm -DCOUNTED_BY -S -g %s -o - | FileCheck %s +// RUN: %clang -emit-llvm -S -g %s -o - | FileCheck %s + +#ifdef COUNTED_BY +#define __counted_by(member) __attribute__((__counted_by__(member))) +#else +#define __counted_by(member) +#endif + +struct { + int num_counters; + long value[] __counted_by(num_counters); +} agent_send_response_port_num; + +// CHECK: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[BT:.*]], elements: ![[ELEMENTS:.*]]) +// CHECK: ![[BT]] = !DIBasicType(name: "long", size: {{.*}}, encoding: DW_ATE_signed) +// CHECK: ![[ELEMENTS]] = !{![[COUNT:.*]]} +// CHECK: ![[COUNT]] = !DISubrange(count: -1) \ No newline at end of file -- GitLab From 56197d732e5d5d158fce2f2dfddf3d0bf0d12525 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Fri, 22 Mar 2024 15:28:54 -0700 Subject: [PATCH 220/933] [BOLT] Skip functions with unsupported Linux kernel features (#86345) Do not overwrite functions with alternative and paravirtual instructions until a proper update support is implemented. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 47 +++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 303e8b18fd32..42df96817275 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -252,11 +252,17 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Paravirtual instruction patch sites. Error readParaInstructions(); + Error rewriteParaInstructions(); Error readBugTable(); - /// Read alternative instruction info from .altinstructions. + /// Do no process functions containing instruction annotated with + /// \p Annotation. + void skipFunctionsWithAnnotation(StringRef Annotation) const; + + /// Handle alternative instruction info from .altinstructions. Error readAltInstructions(); + Error rewriteAltInstructions(); /// Read .pci_fixup Error readPCIFixupTable(); @@ -318,6 +324,12 @@ public: if (Error E = rewriteExceptionTable()) return E; + if (Error E = rewriteAltInstructions()) + return E; + + if (Error E = rewriteParaInstructions()) + return E; + if (Error E = rewriteORCTables()) return E; @@ -1126,6 +1138,31 @@ Error LinuxKernelRewriter::readParaInstructions() { return Error::success(); } +void LinuxKernelRewriter::skipFunctionsWithAnnotation( + StringRef Annotation) const { + for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) { + if (!BC.shouldEmit(BF)) + continue; + for (const BinaryBasicBlock &BB : BF) { + const bool HasAnnotation = llvm::any_of(BB, [&](const MCInst &Inst) { + return BC.MIB->hasAnnotation(Inst, Annotation); + }); + if (HasAnnotation) { + BF.setSimple(false); + break; + } + } + } +} + +Error LinuxKernelRewriter::rewriteParaInstructions() { + // Disable output of functions with paravirtual instructions before the + // rewrite support is complete. + skipFunctionsWithAnnotation("ParaSite"); + + return Error::success(); +} + /// Process __bug_table section. /// This section contains information useful for kernel debugging. /// Each entry in the section is a struct bug_entry that contains a pointer to @@ -1305,6 +1342,14 @@ Error LinuxKernelRewriter::readAltInstructions() { return Error::success(); } +Error LinuxKernelRewriter::rewriteAltInstructions() { + // Disable output of functions with alt instructions before the rewrite + // support is complete. + skipFunctionsWithAnnotation("AltInst"); + + return Error::success(); +} + /// When the Linux kernel needs to handle an error associated with a given PCI /// device, it uses a table stored in .pci_fixup section to locate a fixup code /// specific to the vendor and the problematic device. The section contains a -- GitLab From 51268a57fd4d7f67fe9fdb337f63ec390fa2379a Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Fri, 22 Mar 2024 15:29:26 -0700 Subject: [PATCH 221/933] [BOLT] Enable --keep-nops option for Linux kernel by default (#86349) Preserve nop instructions in the Linux kernel since they could be used for runtime patching. --- bolt/lib/Rewrite/BinaryPassManager.cpp | 2 +- bolt/lib/Rewrite/RewriteInstance.cpp | 4 ++++ bolt/test/X86/linux-alt-instruction.s | 15 +++++++-------- bolt/test/X86/linux-orc.s | 4 ++-- bolt/test/X86/linux-parainstructions.s | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp index 489b33fe1c7c..6c26bb795726 100644 --- a/bolt/lib/Rewrite/BinaryPassManager.cpp +++ b/bolt/lib/Rewrite/BinaryPassManager.cpp @@ -72,7 +72,7 @@ static cl::opt JTFootprintReductionFlag( "instructions at jump sites"), cl::cat(BoltOptCategory)); -static cl::opt +cl::opt KeepNops("keep-nops", cl::desc("keep no-op instructions. By default they are removed."), cl::Hidden, cl::cat(BoltOptCategory)); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 03f4298e817d..2ead51ff6a12 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -81,6 +81,7 @@ extern cl::list HotTextMoveSections; extern cl::opt Hugify; extern cl::opt Instrument; extern cl::opt JumpTables; +extern cl::opt KeepNops; extern cl::list ReorderData; extern cl::opt ReorderFunctions; extern cl::opt TimeBuild; @@ -2031,6 +2032,9 @@ void RewriteInstance::adjustCommandLineOptions() { if (opts::Lite) BC->outs() << "BOLT-INFO: enabling lite mode\n"; + + if (BC->IsLinuxKernel && !opts::KeepNops.getNumOccurrences()) + opts::KeepNops = true; } namespace { diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s index 5dcc6fe3ab0c..2cdf31519682 100644 --- a/bolt/test/X86/linux-alt-instruction.s +++ b/bolt/test/X86/linux-alt-instruction.s @@ -6,8 +6,8 @@ # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops -o %t.out \ -# RUN: --alt-inst-feature-size=2 | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out \ +# RUN: | FileCheck %s ## Older kernels used to have padlen field in alt_instr. Check compatibility. @@ -15,8 +15,8 @@ # RUN: %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops --alt-inst-has-padlen \ -# RUN: -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-has-padlen -o %t.out \ +# RUN: | FileCheck %s ## Check with a larger size of "feature" field in alt_instr. @@ -24,13 +24,12 @@ # RUN: --defsym FEATURE_SIZE_4=1 %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops \ -# RUN: --alt-inst-feature-size=4 -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=4 -o %t.out \ +# RUN: | FileCheck %s ## Check that out-of-bounds read is handled properly. -# RUN: not llvm-bolt %t.exe --print-normalized --keep-nops \ -# RUN: --alt-inst-feature-size=2 -o %t.out +# RUN: not llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out # CHECK: BOLT-INFO: Linux kernel binary detected # CHECK: BOLT-INFO: parsed 2 alternative instruction entries diff --git a/bolt/test/X86/linux-orc.s b/bolt/test/X86/linux-orc.s index 4da19989408e..5f2096278e92 100644 --- a/bolt/test/X86/linux-orc.s +++ b/bolt/test/X86/linux-orc.s @@ -27,7 +27,7 @@ ## Verify ORC bindings to instructions. # RUN: llvm-bolt %t.exe --print-normalized --dump-orc --print-orc -o %t.out \ -# RUN: --bolt-info=0 |& FileCheck %s +# RUN: --keep-nops=0 --bolt-info=0 |& FileCheck %s ## Verify ORC bindings after rewrite. @@ -37,7 +37,7 @@ ## Verify ORC binding after rewrite when some of the functions are skipped. -# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 +# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 --keep-nops=0 # RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized --print-orc \ # RUN: |& FileCheck %s diff --git a/bolt/test/X86/linux-parainstructions.s b/bolt/test/X86/linux-parainstructions.s index 4bdfde5fb7f2..07fca6bbedaf 100644 --- a/bolt/test/X86/linux-parainstructions.s +++ b/bolt/test/X86/linux-parainstructions.s @@ -8,7 +8,7 @@ ## Verify paravirtual bindings to instructions. -# RUN: llvm-bolt %t.exe --print-normalized -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 | FileCheck %s # CHECK: BOLT-INFO: Linux kernel binary detected # CHECK: BOLT-INFO: parsed 2 paravirtual patch sites -- GitLab From 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b Mon Sep 17 00:00:00 2001 From: Usama Hameed Date: Fri, 22 Mar 2024 15:29:36 -0700 Subject: [PATCH 222/933] Get the linker version and pass the it to compiler-rt tests on Darwin. (#86220) The HOST_LINK_VERSION is a hardcoded string in Darwin clang that detects the linker version at configure time. The driver uses this information to build the correct set of arguments for the linker. This patch detects the linker version again during compiler-rt configuration and passes it to the tests. This allows a clang built on a machine with a new linker to run compiler-rt tests on a machine with an old linker. rdar://125198603 --- clang/CMakeLists.txt | 16 ++-------------- cmake/Modules/GetDarwinLinkerVersion.cmake | 19 +++++++++++++++++++ compiler-rt/CMakeLists.txt | 10 ++++++++++ compiler-rt/test/lit.common.cfg.py | 4 ++++ compiler-rt/test/lit.common.configured.in | 1 + 5 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 cmake/Modules/GetDarwinLinkerVersion.cmake diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 761dab8c28c1..ee783d52e4a4 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -15,6 +15,7 @@ endif() # Must go below project(..) include(GNUInstallDirs) +include(GetDarwinLinkerVersion) if(CLANG_BUILT_STANDALONE) set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to") @@ -346,20 +347,7 @@ endif () # Determine HOST_LINK_VERSION on Darwin. set(HOST_LINK_VERSION) if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*") - set(LD_V_OUTPUT) - execute_process( - COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1" - RESULT_VARIABLE HAD_ERROR - OUTPUT_VARIABLE LD_V_OUTPUT - ) - if (HAD_ERROR) - message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}") - endif() - if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*") - string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT}) - elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*") - string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT}) - endif() + get_darwin_linker_version(HOST_LINK_VERSION) message(STATUS "Host linker version: ${HOST_LINK_VERSION}") endif() diff --git a/cmake/Modules/GetDarwinLinkerVersion.cmake b/cmake/Modules/GetDarwinLinkerVersion.cmake new file mode 100644 index 000000000000..c27e50128586 --- /dev/null +++ b/cmake/Modules/GetDarwinLinkerVersion.cmake @@ -0,0 +1,19 @@ +# Get the linker version on Darwin +function(get_darwin_linker_version variable) + set(LINK_VERSION) + set(LD_V_OUTPUT) + execute_process( + COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1" + RESULT_VARIABLE HAD_ERROR + OUTPUT_VARIABLE LD_V_OUTPUT + ) + if (HAD_ERROR) + message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}") + endif() + if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*") + string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT}) + elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*") + string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT}) + endif() + set(${variable} ${LINK_VERSION} PARENT_SCOPE) +endfunction() diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt index d562a6206c00..f4e92f14db85 100644 --- a/compiler-rt/CMakeLists.txt +++ b/compiler-rt/CMakeLists.txt @@ -36,6 +36,7 @@ include(SetPlatformToolchainTools) include(base-config-ix) include(CompilerRTUtils) include(CMakeDependentOption) +include(GetDarwinLinkerVersion) option(COMPILER_RT_BUILD_BUILTINS "Build builtins" ON) mark_as_advanced(COMPILER_RT_BUILD_BUILTINS) @@ -444,6 +445,15 @@ else() set(SANITIZER_USE_SYMBOLS FALSE) endif() +# Get the linker version while configuring compiler-rt and explicitly pass it +# in cflags during testing. This fixes the compiler/linker version mismatch +# issue when running a clang built with a newer Xcode in an older Xcode +set(COMPILER_RT_DARWIN_LINKER_VERSION) +if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*") + get_darwin_linker_version(COMPILER_RT_DARWIN_LINKER_VERSION) + message(STATUS "Host linker version: ${COMPILER_RT_DARWIN_LINKER_VERSION}") +endif() + # Build sanitizer runtimes with debug info. if(MSVC) # Use /Z7 instead of /Zi for the asan runtime. This avoids the LNK4099 diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py index bd9b926c1505..0ac20a9831d9 100644 --- a/compiler-rt/test/lit.common.cfg.py +++ b/compiler-rt/test/lit.common.cfg.py @@ -882,6 +882,10 @@ if config.use_lld and config.has_lld and not config.use_lto: elif config.use_lld and (not config.has_lld): config.unsupported = True +if config.host_os == "Darwin": + if getattr(config, "darwin_linker_version", None): + extra_cflags += ["-mlinker-version=" + config.darwin_linker_version] + # Append any extra flags passed in lit_config append_target_cflags = lit_config.params.get("append_target_cflags", None) if append_target_cflags: diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in index db5d7c598b73..fff5dc6cc750 100644 --- a/compiler-rt/test/lit.common.configured.in +++ b/compiler-rt/test/lit.common.configured.in @@ -51,6 +51,7 @@ set_default("expensive_checks", @LLVM_ENABLE_EXPENSIVE_CHECKS_PYBOOL@) set_default("test_standalone_build_libs", @COMPILER_RT_TEST_STANDALONE_BUILD_LIBS_PYBOOL@) set_default("has_compiler_rt_libatomic", @COMPILER_RT_BUILD_STANDALONE_LIBATOMIC_PYBOOL@) set_default("aarch64_sme", @COMPILER_RT_HAS_AARCH64_SME_PYBOOL@) +set_default("darwin_linker_version", "@COMPILER_RT_DARWIN_LINKER_VERSION@") # True iff the test suite supports ignoring the test compiler's runtime library path # and using `config.compiler_rt_libdir` instead. This only matters when the runtime # library paths differ. -- GitLab From 4406e4a8bd5acadd980d84356b36030cadf9a61d Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 03:22:24 +0500 Subject: [PATCH 223/933] Revert "Missed a null-ptr check in previous PR for Debuginfod testing (#86292)" This reverts commit b1575f9082071702bd6aaa2600ce9fe011a091e9. --- lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index a9956aa9075f..91b8b4a979e0 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), dwp_file_data_sp, dwp_file_data_offset); - if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get())) + if (!ObjectFileELF::classof(dwp_obj_file.get())) return false; // The presence of a debug_cu_index section is the key identifying feature of // a DWP file. Make sure we don't fill in the section list on dwp_obj_file -- GitLab From 7fc2fbb3f1961e0ad0722c2d749ddd6264195a1c Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 03:22:33 +0500 Subject: [PATCH 224/933] Revert "DebugInfoD tests + fixing issues exposed by tests (#85693)" This reverts commit 6d939a6ec69adf284cdbef2034b49fd02ba503fc. This broke following LLDB bots: https://lab.llvm.org/buildbot/#/builders/96/builds/54867 https://lab.llvm.org/buildbot/#/builders/17/builds/50824 --- .../Python/lldbsuite/test/make/Makefile.rules | 33 +-- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 38 ++-- .../Plugins/SymbolLocator/CMakeLists.txt | 7 +- .../SymbolVendor/ELF/SymbolVendorELF.cpp | 30 +-- lldb/test/API/debuginfod/Normal/Makefile | 25 --- .../API/debuginfod/Normal/TestDebuginfod.py | 185 ----------------- lldb/test/API/debuginfod/Normal/main.c | 7 - lldb/test/API/debuginfod/SplitDWARF/Makefile | 28 --- .../SplitDWARF/TestDebuginfodDWP.py | 194 ------------------ lldb/test/API/debuginfod/SplitDWARF/main.c | 7 - 10 files changed, 17 insertions(+), 537 deletions(-) delete mode 100644 lldb/test/API/debuginfod/Normal/Makefile delete mode 100644 lldb/test/API/debuginfod/Normal/TestDebuginfod.py delete mode 100644 lldb/test/API/debuginfod/Normal/main.c delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/Makefile delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/main.c diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules index 75efcde1f040..bfd249ccd43f 100644 --- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules +++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules @@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../ # # GNUWin32 uname gives "windows32" or "server version windows32" while # some versions of MSYS uname return "MSYS_NT*", but most environments -# standardize on "Windows_NT", so we'll make it consistent here. +# standardize on "Windows_NT", so we'll make it consistent here. # When running tests from Visual Studio, the environment variable isn't # inherited all the way down to the process spawned for make. #---------------------------------------------------------------------- @@ -210,12 +210,6 @@ else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" DSYM = $(EXE).debug endif - - ifeq "$(MAKE_DWP)" "YES" - MAKE_DWO := YES - DWP_NAME = $(EXE).dwp - DYLIB_DWP_NAME = $(DYLIB_NAME).dwp - endif endif LIMIT_DEBUG_INFO_FLAGS = @@ -363,7 +357,6 @@ ifneq "$(OS)" "Darwin" OBJCOPY ?= $(call replace_cc_with,objcopy) ARCHIVER ?= $(call replace_cc_with,ar) - DWP ?= $(call replace_cc_with,dwp) override AR = $(ARCHIVER) endif @@ -534,10 +527,6 @@ ifneq "$(CXX)" "" endif endif -ifeq "$(GEN_GNU_BUILD_ID)" "YES" - LDFLAGS += -Wl,--build-id -endif - #---------------------------------------------------------------------- # DYLIB_ONLY variable can be used to skip the building of a.out. # See the sections below regarding dSYM file as well as the building of @@ -576,25 +565,11 @@ else endif else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" -ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" - cp "$(EXE)" "$(EXE).full" -endif $(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)" endif -ifeq "$(MAKE_DWP)" "YES" - $(DWP) -o "$(DWP_NAME)" $(DWOS) -endif endif - -#---------------------------------------------------------------------- -# Support emitting the content of the GNU build-id into a file -# This needs to be used in conjunction with GEN_GNU_BUILD_ID := YES -#---------------------------------------------------------------------- -$(EXE).uuid : $(EXE) - $(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $< - #---------------------------------------------------------------------- # Make the dylib #---------------------------------------------------------------------- @@ -635,15 +610,9 @@ endif else $(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)" ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" - ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" - cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).full" - endif $(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)" endif -ifeq "$(MAKE_DWP)" "YES" - $(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS) -endif endif #---------------------------------------------------------------------- diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 08ce7b82b0c1..5f67658f86ea 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -4377,38 +4377,26 @@ const std::shared_ptr &SymbolFileDWARF::GetDwpSymbolFile() { FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); ModuleSpec module_spec; module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec(); - FileSpec dwp_filespec; for (const auto &symfile : symfiles.files()) { module_spec.GetSymbolFileSpec() = FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle()); LLDB_LOG(log, "Searching for DWP using: \"{0}\"", module_spec.GetSymbolFileSpec()); - dwp_filespec = + FileSpec dwp_filespec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); if (FileSystem::Instance().Exists(dwp_filespec)) { - break; - } - } - if (!FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "No DWP file found locally"); - // Fill in the UUID for the module we're trying to match for, so we can - // find the correct DWP file, as the Debuginfod plugin uses *only* this - // data to correctly match the DWP file with the binary. - module_spec.GetUUID() = m_objfile_sp->GetUUID(); - dwp_filespec = - PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - } - if (FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - GetObjectFile()->GetModule(), &dwp_filespec, 0, - FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, - dwp_file_data_offset); - if (dwp_obj_file) { - m_dwp_symfile = std::make_shared( - *this, dwp_obj_file, DIERef::k_file_index_mask); + LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + GetObjectFile()->GetModule(), &dwp_filespec, 0, + FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, + dwp_file_data_offset); + if (dwp_obj_file) { + m_dwp_symfile = std::make_shared( + *this, dwp_obj_file, DIERef::k_file_index_mask); + break; + } } } if (!m_dwp_symfile) { diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt index 3367022639ab..ca969626f4ff 100644 --- a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt +++ b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt @@ -1,10 +1,5 @@ -# Order matters here: the first symbol locator prevents further searching. -# For DWARF binaries that are both stripped and split, the Default plugin -# will return the stripped binary when asked for the ObjectFile, which then -# prevents an unstripped binary from being requested from the Debuginfod -# provider. -add_subdirectory(Debuginfod) add_subdirectory(Default) if (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_subdirectory(DebugSymbols) endif() +add_subdirectory(Debuginfod) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index 91b8b4a979e0..b5fe35d71032 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -44,25 +44,6 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() { "executables."; } -// If this is needed elsewhere, it can be exported/moved. -static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, - const FileSpec &file_spec) { - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - // Try to create an ObjectFile from the file_spec. - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), - dwp_file_data_sp, dwp_file_data_offset); - if (!ObjectFileELF::classof(dwp_obj_file.get())) - return false; - // The presence of a debug_cu_index section is the key identifying feature of - // a DWP file. Make sure we don't fill in the section list on dwp_obj_file - // (by calling GetSectionList(false)) as this is invoked before we may have - // all the symbol files collected and available. - return dwp_obj_file && dwp_obj_file->GetSectionList(false)->FindSectionByType( - eSectionTypeDWARFDebugCuIndex, false); -} - // CreateInstance // // Platforms can register a callback to use when creating symbol vendors to @@ -106,15 +87,8 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp, FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); FileSpec dsym_fspec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) { - // If we have a stripped binary or if we got a DWP file, we should prefer - // symbols in the executable acquired through a plugin. - ModuleSpec unstripped_spec = - PluginManager::LocateExecutableObjectFile(module_spec); - if (!unstripped_spec) - return nullptr; - dsym_fspec = unstripped_spec.GetFileSpec(); - } + if (!dsym_fspec) + return nullptr; DataBufferSP dsym_file_data_sp; lldb::offset_t dsym_file_data_offset = 0; diff --git a/lldb/test/API/debuginfod/Normal/Makefile b/lldb/test/API/debuginfod/Normal/Makefile deleted file mode 100644 index bd2fa623df47..000000000000 --- a/lldb/test/API/debuginfod/Normal/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -C_SOURCES := main.c - -# For normal (non DWP) Debuginfod tests, we need: - -# * The "full" binary: a.out.debug -# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and -# SPLIT_DEBUG_SYMBOLS set to YES - -# * The stripped binary (a.out) -# Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES - -# * The 'only-keep-debug' binary (a.out.dbg) -# Produced below - -# * The .uuid file (for a little easier testing code) -# Produced below - -# Don't strip the debug info from a.out: -SPLIT_DEBUG_SYMBOLS := YES -SAVE_FULL_DEBUG_BINARY := YES -GEN_GNU_BUILD_ID := YES - -all: a.out.uuid a.out - -include Makefile.rules diff --git a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py deleted file mode 100644 index eb5efe83c17a..000000000000 --- a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py +++ /dev/null @@ -1,185 +0,0 @@ -import os -import shutil -import tempfile -import struct - -import lldb -from lldbsuite.test.decorators import * -import lldbsuite.test.lldbutil as lldbutil -from lldbsuite.test.lldbtest import * - - -def getUUID(aoutuuid): - """ - Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped - to a file already, as part of the build. - """ - with open(aoutuuid, "rb") as f: - data = f.read(36) - if len(data) != 36: - return None - header = struct.unpack_from("<4I", data) - if len(header) != 4: - return None - # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': - if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: - return None - return data[16:].hex() - - -""" -Test support for the DebugInfoD network symbol acquisition protocol. -This one is for simple / no split-dwarf scenarios. - -For no-split-dwarf scenarios, there are 2 variations: -1 - A stripped binary with it's corresponding unstripped binary: -2 - A stripped binary with a corresponding --only-keep-debug symbols file -""" - - -@skipUnlessPlatform(["linux", "freebsd"]) -class DebugInfodTests(TestBase): - # No need to try every flavor of debug inf. - NO_DEBUG_INFO_TESTCASE = True - - def test_normal_no_symbols(self): - """ - Validate behavior with no symbols or symbol locator. - ('baseline negative' behavior) - """ - test_root = self.config_test(["a.out"]) - self.try_breakpoint(False) - - def test_normal_default(self): - """ - Validate behavior with symbols, but no symbol locator. - ('baseline positive' behavior) - """ - test_root = self.config_test(["a.out", "a.out.debug"]) - self.try_breakpoint(True) - - def test_debuginfod_symbols(self): - """ - Test behavior with the full binary available from Debuginfod as - 'debuginfo' from the plug-in. - """ - test_root = self.config_test(["a.out"], "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_executable(self): - """ - Test behavior with the full binary available from Debuginfod as - 'executable' from the plug-in. - """ - test_root = self.config_test(["a.out"], None, "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_okd_symbols(self): - """ - Test behavior with the 'only-keep-debug' symbols available from Debuginfod. - """ - test_root = self.config_test(["a.out"], "a.out.debug") - self.try_breakpoint(True) - - def try_breakpoint(self, should_have_loc): - """ - This function creates a target from self.aout, sets a function-name - breakpoint, and checks to see if we have a file/line location, - as a way to validate that the symbols have been loaded. - should_have_loc specifies if we're testing that symbols have or - haven't been loaded. - """ - target = self.dbg.CreateTarget(self.aout) - self.assertTrue(target and target.IsValid(), "Target is valid") - - bp = target.BreakpointCreateByName("func") - self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") - self.assertEqual(bp.GetNumLocations(), 1) - - loc = bp.GetLocationAtIndex(0) - self.assertTrue(loc and loc.IsValid(), "Location is valid") - addr = loc.GetAddress() - self.assertTrue(addr and addr.IsValid(), "Loc address is valid") - line_entry = addr.GetLineEntry() - self.assertEqual( - should_have_loc, - line_entry != None and line_entry.IsValid(), - "Loc line entry is valid", - ) - if should_have_loc: - self.assertEqual(line_entry.GetLine(), 4) - self.assertEqual( - line_entry.GetFileSpec().GetFilename(), - self.main_source_file.GetFilename(), - ) - self.dbg.DeleteTarget(target) - shutil.rmtree(self.tmp_dir) - - def config_test(self, local_files, debuginfo=None, executable=None): - """ - Set up a test with local_files[] copied to a different location - so that we control which files are, or are not, found in the file system. - Also, create a stand-alone file-system 'hosted' debuginfod server with the - provided debuginfo and executable files (if they exist) - - Make the filesystem look like: - - /tmp//test/[local_files] - - /tmp//cache (for lldb to use as a temp cache) - - /tmp//buildid//executable -> - /tmp//buildid//debuginfo -> - Returns the /tmp/ path - """ - - self.build() - - uuid = getUUID(self.getBuildArtifact("a.out.uuid")) - - self.main_source_file = lldb.SBFileSpec("main.c") - self.tmp_dir = tempfile.mkdtemp() - test_dir = os.path.join(self.tmp_dir, "test") - os.makedirs(test_dir) - - self.aout = "" - # Copy the files used by the test: - for f in local_files: - shutil.copy(self.getBuildArtifact(f), test_dir) - # The first item is the binary to be used for the test - if self.aout == "": - self.aout = os.path.join(test_dir, f) - - use_debuginfod = debuginfo != None or executable != None - - # Populated the 'file://... mocked' Debuginfod server: - if use_debuginfod: - os.makedirs(os.path.join(self.tmp_dir, "cache")) - uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) - os.makedirs(uuid_dir) - if debuginfo: - shutil.copy( - self.getBuildArtifact(debuginfo), - os.path.join(uuid_dir, "debuginfo"), - ) - if executable: - shutil.copy( - self.getBuildArtifact(executable), - os.path.join(uuid_dir, "executable"), - ) - - # Configure LLDB for the test: - self.runCmd( - "settings set symbols.enable-external-lookup %s" - % str(use_debuginfod).lower() - ) - self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") - if use_debuginfod: - self.runCmd( - "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" - % self.tmp_dir - ) - self.runCmd( - "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" - % self.tmp_dir - ) diff --git a/lldb/test/API/debuginfod/Normal/main.c b/lldb/test/API/debuginfod/Normal/main.c deleted file mode 100644 index 4c7184609b45..000000000000 --- a/lldb/test/API/debuginfod/Normal/main.c +++ /dev/null @@ -1,7 +0,0 @@ -// This is a dump little pair of test files - -int func(int argc, const char *argv[]) { - return (argc + 1) * (argv[argc][0] + 2); -} - -int main(int argc, const char *argv[]) { return func(0, argv); } diff --git a/lldb/test/API/debuginfod/SplitDWARF/Makefile b/lldb/test/API/debuginfod/SplitDWARF/Makefile deleted file mode 100644 index 266d74cf9062..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -C_SOURCES := main.c - -# For split-dwarf Debuginfod tests, we need: - -# * A .DWP file (a.out.dwp) -# Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES - -# * The "full" binary: it's missing things that live in .dwo's (a.out.debug) -# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and -# SPLIT_DEBUG_SYMBOLS set to YES - -# * The stripped binary (a.out) -# Produced by Makefile.rules - -# * The 'only-keep-debug' binary (a.out.dbg) -# Produced below - -# * The .uuid file (for a little easier testing code) -# Produced here in the rule below - -MAKE_DWP := YES -SPLIT_DEBUG_SYMBOLS := YES -SAVE_FULL_DEBUG_BINARY := YES -GEN_GNU_BUILD_ID := YES - -all: a.out.uuid a.out - -include Makefile.rules diff --git a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py deleted file mode 100644 index 09f91b6f1c6c..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Test support for the DebugInfoD network symbol acquisition protocol. -""" -import os -import shutil -import tempfile -import struct - -import lldb -from lldbsuite.test.decorators import * -import lldbsuite.test.lldbutil as lldbutil -from lldbsuite.test.lldbtest import * - - -def getUUID(aoutuuid): - """ - Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped - to a file already, as part of the build. - """ - with open(aoutuuid, "rb") as f: - data = f.read(36) - if len(data) != 36: - return None - header = struct.unpack_from("<4I", data) - if len(header) != 4: - return None - # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': - if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: - return None - return data[16:].hex() - - -""" -Test support for the DebugInfoD network symbol acquisition protocol. -This file is for split-dwarf (dwp) scenarios. - -1 - A split binary target with it's corresponding DWP file -2 - A stripped, split binary target with an unstripped binary and a DWP file -3 - A stripped, split binary target with an --only-keep-debug symbols file and a DWP file -""" - - -@skipUnlessPlatform(["linux", "freebsd"]) -class DebugInfodDWPTests(TestBase): - # No need to try every flavor of debug inf. - NO_DEBUG_INFO_TESTCASE = True - - def test_normal_stripped(self): - """ - Validate behavior with a stripped binary, no symbols or symbol locator. - """ - self.config_test(["a.out"]) - self.try_breakpoint(False) - - def test_normal_stripped_split_with_dwp(self): - """ - Validate behavior with symbols, but no symbol locator. - """ - self.config_test(["a.out", "a.out.debug", "a.out.dwp"]) - self.try_breakpoint(True) - - def test_normal_stripped_only_dwp(self): - """ - Validate behavior *with* dwp symbols only, but missing other symbols, - but no symbol locator. This shouldn't work: without the other symbols - DWO's appear mostly useless. - """ - self.config_test(["a.out", "a.out.dwp"]) - self.try_breakpoint(False) - - def test_debuginfod_dwp_from_service(self): - """ - Test behavior with the unstripped binary, and DWP from the service. - """ - self.config_test(["a.out.debug"], "a.out.dwp") - self.try_breakpoint(True) - - def test_debuginfod_both_symfiles_from_service(self): - """ - Test behavior with a stripped binary, with the unstripped binary and - dwp symbols from Debuginfod. - """ - self.config_test(["a.out"], "a.out.dwp", "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_both_okd_symfiles_from_service(self): - """ - Test behavior with both the only-keep-debug symbols and the dwp symbols - from Debuginfod. - """ - self.config_test(["a.out"], "a.out.dwp", "a.out.debug") - self.try_breakpoint(True) - - def try_breakpoint(self, should_have_loc): - """ - This function creates a target from self.aout, sets a function-name - breakpoint, and checks to see if we have a file/line location, - as a way to validate that the symbols have been loaded. - should_have_loc specifies if we're testing that symbols have or - haven't been loaded. - """ - target = self.dbg.CreateTarget(self.aout) - self.assertTrue(target and target.IsValid(), "Target is valid") - - bp = target.BreakpointCreateByName("func") - self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") - self.assertEqual(bp.GetNumLocations(), 1) - - loc = bp.GetLocationAtIndex(0) - self.assertTrue(loc and loc.IsValid(), "Location is valid") - addr = loc.GetAddress() - self.assertTrue(addr and addr.IsValid(), "Loc address is valid") - line_entry = addr.GetLineEntry() - self.assertEqual( - should_have_loc, - line_entry != None and line_entry.IsValid(), - "Loc line entry is valid", - ) - if should_have_loc: - self.assertEqual(line_entry.GetLine(), 4) - self.assertEqual( - line_entry.GetFileSpec().GetFilename(), - self.main_source_file.GetFilename(), - ) - self.dbg.DeleteTarget(target) - shutil.rmtree(self.tmp_dir) - - def config_test(self, local_files, debuginfo=None, executable=None): - """ - Set up a test with local_files[] copied to a different location - so that we control which files are, or are not, found in the file system. - Also, create a stand-alone file-system 'hosted' debuginfod server with the - provided debuginfo and executable files (if they exist) - - Make the filesystem look like: - - /tmp//test/[local_files] - - /tmp//cache (for lldb to use as a temp cache) - - /tmp//buildid//executable -> - /tmp//buildid//debuginfo -> - Returns the /tmp/ path - """ - - self.build() - - uuid = getUUID(self.getBuildArtifact("a.out.uuid")) - - self.main_source_file = lldb.SBFileSpec("main.c") - self.tmp_dir = tempfile.mkdtemp() - self.test_dir = os.path.join(self.tmp_dir, "test") - os.makedirs(self.test_dir) - - self.aout = "" - # Copy the files used by the test: - for f in local_files: - shutil.copy(self.getBuildArtifact(f), self.test_dir) - if self.aout == "": - self.aout = os.path.join(self.test_dir, f) - - use_debuginfod = debuginfo != None or executable != None - - # Populated the 'file://... mocked' Debuginfod server: - if use_debuginfod: - os.makedirs(os.path.join(self.tmp_dir, "cache")) - uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) - os.makedirs(uuid_dir) - if debuginfo: - shutil.copy( - self.getBuildArtifact(debuginfo), - os.path.join(uuid_dir, "debuginfo"), - ) - if executable: - shutil.copy( - self.getBuildArtifact(executable), - os.path.join(uuid_dir, "executable"), - ) - os.remove(self.getBuildArtifact("main.dwo")) - # Configure LLDB for the test: - self.runCmd( - "settings set symbols.enable-external-lookup %s" - % str(use_debuginfod).lower() - ) - self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") - if use_debuginfod: - self.runCmd( - "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" - % self.tmp_dir - ) - self.runCmd( - "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" - % self.tmp_dir - ) diff --git a/lldb/test/API/debuginfod/SplitDWARF/main.c b/lldb/test/API/debuginfod/SplitDWARF/main.c deleted file mode 100644 index 4c7184609b45..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/main.c +++ /dev/null @@ -1,7 +0,0 @@ -// This is a dump little pair of test files - -int func(int argc, const char *argv[]) { - return (argc + 1) * (argv[argc][0] + 2); -} - -int main(int argc, const char *argv[]) { return func(0, argv); } -- GitLab From 00248754176d74aed2e0785d9982a5ea8e91a71a Mon Sep 17 00:00:00 2001 From: "Daniel M. Katz" Date: Fri, 22 Mar 2024 19:08:02 -0400 Subject: [PATCH 225/933] [Clang] Raise an error on namespace aliases with qualified names. (#86122) --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Basic/DiagnosticParseKinds.td | 2 ++ clang/lib/Parse/ParseDeclCXX.cpp | 8 ++++++++ clang/test/SemaCXX/namespace-alias.cpp | 2 ++ 4 files changed, 14 insertions(+) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index d6e179ca9d69..8054d90fc70f 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -437,6 +437,8 @@ Bug Fixes to C++ Support - Clang's __builtin_bit_cast will now produce a constant value for records with empty bases. See: (#GH82383) - Fix a crash when instantiating a lambda that captures ``this`` outside of its context. Fixes (#GH85343). +- Fix an issue where a namespace alias could be defined using a qualified name (all name components + following the first `::` were ignored). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index 48de5e2ef5f4..46a44418a315 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -268,6 +268,8 @@ def err_expected_semi_after_namespace_name : Error< "expected ';' after namespace name">; def err_unexpected_namespace_attributes_alias : Error< "attributes cannot be specified on namespace alias">; +def err_unexpected_qualified_namespace_alias : Error< + "namespace alias must be a single identifier">; def err_unexpected_nested_namespace_attribute : Error< "attributes cannot be specified on a nested namespace definition">; def err_inline_namespace_alias : Error<"namespace alias cannot be inline">; diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 77d2382ea6d9..63fe678cbb29 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -140,6 +140,14 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context, SkipUntil(tok::semi); return nullptr; } + if (!ExtraNSs.empty()) { + Diag(ExtraNSs.front().NamespaceLoc, + diag::err_unexpected_qualified_namespace_alias) + << SourceRange(ExtraNSs.front().NamespaceLoc, + ExtraNSs.back().IdentLoc); + SkipUntil(tok::semi); + return nullptr; + } if (attrLoc.isValid()) Diag(attrLoc, diag::err_unexpected_namespace_attributes_alias); if (InlineLoc.isValid()) diff --git a/clang/test/SemaCXX/namespace-alias.cpp b/clang/test/SemaCXX/namespace-alias.cpp index 281ee9962e8b..591957a657c0 100644 --- a/clang/test/SemaCXX/namespace-alias.cpp +++ b/clang/test/SemaCXX/namespace-alias.cpp @@ -47,6 +47,8 @@ namespace I { namespace A1 { int i; } namespace A2 = A1; + + namespace A3::extra::specifiers = A2; // expected-error {{alias must be a single identifier}} } int f() { -- GitLab From 5d187898f625cc54310f51b278b36ad6a97104ee Mon Sep 17 00:00:00 2001 From: Fabian Tschopp Date: Sat, 23 Mar 2024 00:09:11 +0100 Subject: [PATCH 226/933] [mlir][inliner] Return early if the inliningThreshold is 0U or -1U. (#86287) Computing the inlinling profitability can be costly due to walking the graph when counting the number of operations. This PR addresses that by returning early if the threshold is set to never or always inline. --- mlir/lib/Transforms/InlinerPass.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp index 08d8dbf73a6a..9a7d5403a95d 100644 --- a/mlir/lib/Transforms/InlinerPass.cpp +++ b/mlir/lib/Transforms/InlinerPass.cpp @@ -93,12 +93,19 @@ InlinerPass::InlinerPass(std::function defaultPipeline, // Return true if the inlining ratio does not exceed the threshold. static bool isProfitableToInline(const Inliner::ResolvedCall &resolvedCall, unsigned inliningThreshold) { + // Return early, ratio <= 0U will always be false. + if (inliningThreshold == 0U) + return false; + // Return early, ratio <= -1U will always be true. + if (inliningThreshold == -1U) + return true; + Region *callerRegion = resolvedCall.sourceNode->getCallableRegion(); Region *calleeRegion = resolvedCall.targetNode->getCallableRegion(); // We should not get external nodes here, but just return true // for now to preserve the original behavior of the inliner pass. - if (!calleeRegion || !calleeRegion) + if (!callerRegion || !calleeRegion) return true; auto countOps = [](Region *region) { -- GitLab From b768a8c1db85b9e84fd8b356570a3a8fbe37acf6 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Fri, 22 Mar 2024 16:09:34 -0700 Subject: [PATCH 227/933] [clang][deps] Lazy dependency directives (#86347) Since b4c83a13f664582015ea22924b9a0c6290d41f5b, `Preprocessor` and `Lexer` are aware of the concept of scanning dependency directives. This makes it possible to scan for them on-demand rather than eagerly on the first filesystem operation (open, or even just stat). This might improve performance, but is also necessary for the "PCH as module" mode. Some precompiled header sources use the ".pch" file extension, which means they were not getting scanned for dependency directives. This was okay when the PCH was the main input file in a separate scan step, because there we just lex the file in a scanning-specific frontend action. But when such source gets treated as a module implicitly loaded from a TU, it will get compiled as any other module - with Sema - which will result in compilation errors. (See attached test case.) rdar://107663951 --- .../DependencyScanningFilesystem.h | 20 +++--- .../DependencyScanningFilesystem.cpp | 63 +++++-------------- .../DependencyScanningWorker.cpp | 3 +- clang/test/ClangScanDeps/modules-extension.c | 33 ++++++++++ 4 files changed, 60 insertions(+), 59 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-extension.c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index 846fdc725397..9a522a3e2fe2 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -242,6 +242,8 @@ class EntryRef { /// The underlying cached entry. const CachedFileSystemEntry &Entry; + friend class DependencyScanningWorkerFilesystem; + public: EntryRef(StringRef Name, const CachedFileSystemEntry &Entry) : Filename(Name), Entry(Entry) {} @@ -300,14 +302,15 @@ public: /// /// Attempts to use the local and shared caches first, then falls back to /// using the underlying filesystem. - llvm::ErrorOr - getOrCreateFileSystemEntry(StringRef Filename, - bool DisableDirectivesScanning = false); + llvm::ErrorOr getOrCreateFileSystemEntry(StringRef Filename); -private: - /// Check whether the file should be scanned for preprocessor directives. - bool shouldScanForDirectives(StringRef Filename); + /// Ensure the directive tokens are populated for this file entry. + /// + /// Returns true if the directive tokens are populated for this file entry, + /// false if not (i.e. this entry is not a file or its scan fails). + bool ensureDirectiveTokensArePopulated(EntryRef Entry); +private: /// For a filename that's not yet associated with any entry in the caches, /// uses the underlying filesystem to either look up the entry based in the /// shared cache indexed by unique ID, or creates new entry from scratch. @@ -317,11 +320,6 @@ private: computeAndStoreResult(StringRef OriginalFilename, StringRef FilenameForLookup); - /// Scan for preprocessor directives for the given entry if necessary and - /// returns a wrapper object with reference semantics. - EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry, - StringRef Filename, bool Disable); - /// Represents a filesystem entry that has been stat-ed (and potentially read) /// and that's about to be inserted into the cache as `CachedFileSystemEntry`. struct TentativeEntry { diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 1b750cec41e1..9b7812a1adb9 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -41,24 +41,25 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) { return TentativeEntry(Stat, std::move(Buffer)); } -EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( - const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) { - if (Entry.isError() || Entry.isDirectory() || Disable || - !shouldScanForDirectives(Filename)) - return EntryRef(Filename, Entry); +bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated( + EntryRef Ref) { + auto &Entry = Ref.Entry; + + if (Entry.isError() || Entry.isDirectory()) + return false; CachedFileContents *Contents = Entry.getCachedContents(); assert(Contents && "contents not initialized"); // Double-checked locking. if (Contents->DepDirectives.load()) - return EntryRef(Filename, Entry); + return true; std::lock_guard GuardLock(Contents->ValueLock); // Double-checked locking. if (Contents->DepDirectives.load()) - return EntryRef(Filename, Entry); + return true; SmallVector Directives; // Scan the file for preprocessor directives that might affect the @@ -69,16 +70,16 @@ EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( Contents->DepDirectiveTokens.clear(); // FIXME: Propagate the diagnostic if desired by the client. Contents->DepDirectives.store(new std::optional()); - return EntryRef(Filename, Entry); + return false; } // This function performed double-checked locking using `DepDirectives`. // Assigning it must be the last thing this function does, otherwise other - // threads may skip the - // critical section (`DepDirectives != nullptr`), leading to a data race. + // threads may skip the critical section (`DepDirectives != nullptr`), leading + // to a data race. Contents->DepDirectives.store( new std::optional(std::move(Directives))); - return EntryRef(Filename, Entry); + return true; } DependencyScanningFilesystemSharedCache:: @@ -161,34 +162,11 @@ DependencyScanningFilesystemSharedCache::CacheShard:: return *EntriesByFilename.insert({Filename, &Entry}).first->getValue(); } -/// Whitelist file extensions that should be minimized, treating no extension as -/// a source file that should be minimized. -/// -/// This is kinda hacky, it would be better if we knew what kind of file Clang -/// was expecting instead. -static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) { - StringRef Ext = llvm::sys::path::extension(Filename); - if (Ext.empty()) - return true; // C++ standard library - return llvm::StringSwitch(Ext) - .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) - .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) - .CasesLower(".m", ".mm", true) - .CasesLower(".i", ".ii", ".mi", ".mmi", true) - .CasesLower(".def", ".inc", true) - .Default(false); -} - static bool shouldCacheStatFailures(StringRef Filename) { StringRef Ext = llvm::sys::path::extension(Filename); if (Ext.empty()) return false; // This may be the module cache directory. - // Only cache stat failures on files that are not expected to change during - // the build. - StringRef FName = llvm::sys::path::filename(Filename); - if (FName == "module.modulemap" || FName == "module.map") - return true; - return shouldScanForDirectivesBasedOnExtension(Filename); + return true; } DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( @@ -201,11 +179,6 @@ DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( updateWorkingDirForCacheLookup(); } -bool DependencyScanningWorkerFilesystem::shouldScanForDirectives( - StringRef Filename) { - return shouldScanForDirectivesBasedOnExtension(Filename); -} - const CachedFileSystemEntry & DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID( TentativeEntry TEntry) { @@ -259,7 +232,7 @@ DependencyScanningWorkerFilesystem::computeAndStoreResult( llvm::ErrorOr DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( - StringRef OriginalFilename, bool DisableDirectivesScanning) { + StringRef OriginalFilename) { StringRef FilenameForLookup; SmallString<256> PathBuf; if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) { @@ -276,15 +249,11 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup)); if (const auto *Entry = findEntryByFilenameWithWriteThrough(FilenameForLookup)) - return scanForDirectivesIfNecessary(*Entry, OriginalFilename, - DisableDirectivesScanning) - .unwrapError(); + return EntryRef(OriginalFilename, *Entry).unwrapError(); auto MaybeEntry = computeAndStoreResult(OriginalFilename, FilenameForLookup); if (!MaybeEntry) return MaybeEntry.getError(); - return scanForDirectivesIfNecessary(*MaybeEntry, OriginalFilename, - DisableDirectivesScanning) - .unwrapError(); + return EntryRef(OriginalFilename, *MaybeEntry).unwrapError(); } llvm::ErrorOr diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 76f3d950a13b..33b43417a661 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -372,7 +372,8 @@ public: -> std::optional> { if (llvm::ErrorOr Entry = LocalDepFS->getOrCreateFileSystemEntry(File.getName())) - return Entry->getDirectiveTokens(); + if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry)) + return Entry->getDirectiveTokens(); return std::nullopt; }; } diff --git a/clang/test/ClangScanDeps/modules-extension.c b/clang/test/ClangScanDeps/modules-extension.c new file mode 100644 index 000000000000..0f27f608440f --- /dev/null +++ b/clang/test/ClangScanDeps/modules-extension.c @@ -0,0 +1,33 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t + +// This test checks that source files with uncommon extensions still undergo +// dependency directives scan. If header.pch would not and b.h would, the scan +// would fail when parsing `void function(B)` and not knowing the symbol B. + +//--- module.modulemap +module __PCH { header "header.pch" } +module B { header "b.h" } + +//--- header.pch +#include "b.h" +void function(B); + +//--- b.h +typedef int B; + +//--- tu.c +int main() { + function(0); + return 0; +} + +//--- cdb.json.in +[{ + "directory": "DIR", + "file": "DIR/tu.c", + "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -fimplicit-module-maps -include DIR/header.pch" +}] + +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/deps.json -- GitLab From af63c6e5d08fcaeacaeee68aa0a1cda71d9a7549 Mon Sep 17 00:00:00 2001 From: Philipp Tomsich Date: Sat, 23 Mar 2024 00:29:58 +0100 Subject: [PATCH 228/933] [AArch64] Adjust ROBsize for Ampere1/Ampere1A (NFC) (#86330) To align more closely with common usage, we now use the size of the reorder-buffer for MicroOpBufferSize instead of the entries of the global micro-op scheduler. --- llvm/lib/Target/AArch64/AArch64SchedAmpere1.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td index cf9f50c2784b..269f4ec5e5fb 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td +++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td @@ -18,7 +18,7 @@ def Ampere1Model : SchedMachineModel { let IssueWidth = 4; // 4-way decode and dispatch - let MicroOpBufferSize = 174; // micro-op re-order buffer size + let MicroOpBufferSize = 192; // re-order buffer size let LoadLatency = 4; // Optimistic load latency let MispredictPenalty = 10; // Branch mispredict penalty let LoopMicroOpBufferSize = 32; // Instruction queue size -- GitLab From b621269d4a4c08269b1b2d46f277d1918d3dab62 Mon Sep 17 00:00:00 2001 From: Philipp Tomsich Date: Sat, 23 Mar 2024 00:30:12 +0100 Subject: [PATCH 229/933] [AArch64] Adjust ROBsize for Ampere1B (NFC) (#86331) To align more closely with common usage, we now use the size of the reorder-buffer for MicroOpBufferSize instead of the entries of the global micro-op scheduler. --- llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td index 9c4f000cf351..67f8593f1577 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td +++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td @@ -18,7 +18,7 @@ def Ampere1BModel : SchedMachineModel { let IssueWidth = 12; // Maximum micro-ops dispatch rate. - let MicroOpBufferSize = 192; // micro-op re-order buffer size + let MicroOpBufferSize = 208; // micro-op re-order buffer size let LoadLatency = 3; // Optimistic load latency let MispredictPenalty = 10; // Branch mispredict penalty let LoopMicroOpBufferSize = 32; // Instruction queue size -- GitLab From c3747883a0bf34d271bc89dbfc60590adf75d999 Mon Sep 17 00:00:00 2001 From: Xiaoyang Liu Date: Fri, 22 Mar 2024 16:32:02 -0700 Subject: [PATCH 230/933] [libc++][ranges] use `static operator()` for C++23 ranges (#86052) ## Abstract This pull request converts the `operator()` of all CPOs and niebloids related to C++23 ranges to `static`. ## Motivation In `libc++`, CPOs and niebloids are implemented as function objects. Currently, the `operator()` for such a function object is a `const`-qualified member function. This means that even if the function object is has no data members, an extra register is used to pass in the `this` pointer when calling `operator()`, unless the compiler can inline the function call. Declaraing `operator()` as `static` would optimize away the unnecessary `this` pointer passing for stateless function objects, since there is no object instance state that needs to be accessed. ## Reference - [P1169R4: static `operator()`](https://wg21.link/P1169R4) --- libcxx/include/__algorithm/ranges_ends_with.h | 6 +++--- libcxx/include/__algorithm/ranges_starts_with.h | 8 ++++---- libcxx/include/__ranges/as_rvalue_view.h | 16 ++++++++-------- libcxx/include/__ranges/repeat_view.h | 5 ++--- libcxx/include/__ranges/to.h | 4 ++-- libcxx/include/__ranges/zip_view.h | 8 ++++---- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/libcxx/include/__algorithm/ranges_ends_with.h b/libcxx/include/__algorithm/ranges_ends_with.h index c2a3cae9f3b1..bb01918326b8 100644 --- a/libcxx/include/__algorithm/ranges_ends_with.h +++ b/libcxx/include/__algorithm/ranges_ends_with.h @@ -39,7 +39,7 @@ namespace ranges { namespace __ends_with { struct __fn { template - static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl_bidirectional( + _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl_bidirectional( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, @@ -56,7 +56,7 @@ struct __fn { } template - static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl( + _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, @@ -65,7 +65,7 @@ struct __fn { _Proj1& __proj1, _Proj2& __proj2) { if constexpr (std::bidirectional_iterator<_Sent1> && std::bidirectional_iterator<_Sent2> && - (!std::random_access_iterator<_Sent1>)&&(!std::random_access_iterator<_Sent2>)) { + (!std::random_access_iterator<_Sent1>) && (!std::random_access_iterator<_Sent2>)) { return __ends_with_fn_impl_bidirectional(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2); } else { diff --git a/libcxx/include/__algorithm/ranges_starts_with.h b/libcxx/include/__algorithm/ranges_starts_with.h index 90e184aa9bcc..7ba8af13a8d1 100644 --- a/libcxx/include/__algorithm/ranges_starts_with.h +++ b/libcxx/include/__algorithm/ranges_starts_with.h @@ -42,14 +42,14 @@ struct __fn { class _Proj1 = identity, class _Proj2 = identity> requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()( + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool operator()( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred __pred = {}, _Proj1 __proj1 = {}, - _Proj2 __proj2 = {}) const { + _Proj2 __proj2 = {}) { return __mismatch::__fn::__go( std::move(__first1), std::move(__last1), @@ -67,8 +67,8 @@ struct __fn { class _Proj1 = identity, class _Proj2 = identity> requires indirectly_comparable, iterator_t<_Range2>, _Pred, _Proj1, _Proj2> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()( - _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool + operator()(_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) { return __mismatch::__fn::__go( ranges::begin(__range1), ranges::end(__range1), diff --git a/libcxx/include/__ranges/as_rvalue_view.h b/libcxx/include/__ranges/as_rvalue_view.h index 295aa94ed9fe..2fc272e798d6 100644 --- a/libcxx/include/__ranges/as_rvalue_view.h +++ b/libcxx/include/__ranges/as_rvalue_view.h @@ -111,18 +111,18 @@ namespace views { namespace __as_rvalue { struct __fn : __range_adaptor_closure<__fn> { template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const - noexcept(noexcept(/**/ as_rvalue_view(std::forward<_Range>(__range)))) - -> decltype(/*--*/ as_rvalue_view(std::forward<_Range>(__range))) { - return /*-------------*/ as_rvalue_view(std::forward<_Range>(__range)); + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Range&& __range) noexcept(noexcept(as_rvalue_view(std::forward<_Range>(__range)))) + -> decltype(/*--------------------------*/ as_rvalue_view(std::forward<_Range>(__range))) { + return /*---------------------------------*/ as_rvalue_view(std::forward<_Range>(__range)); } template requires same_as, range_reference_t<_Range>> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const - noexcept(noexcept(/**/ views::all(std::forward<_Range>(__range)))) - -> decltype(/*--*/ views::all(std::forward<_Range>(__range))) { - return /*-------------*/ views::all(std::forward<_Range>(__range)); + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Range&& __range) noexcept(noexcept(views::all(std::forward<_Range>(__range)))) + -> decltype(/*--------------------------*/ views::all(std::forward<_Range>(__range))) { + return /*---------------------------------*/ views::all(std::forward<_Range>(__range)); } }; } // namespace __as_rvalue diff --git a/libcxx/include/__ranges/repeat_view.h b/libcxx/include/__ranges/repeat_view.h index 620a26454972..5caea757a393 100644 --- a/libcxx/include/__ranges/repeat_view.h +++ b/libcxx/include/__ranges/repeat_view.h @@ -229,14 +229,13 @@ namespace views { namespace __repeat { struct __fn { template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value) const + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value) noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value)))) -> decltype( ranges::repeat_view(std::forward<_Tp>(__value))) { return ranges::repeat_view(std::forward<_Tp>(__value)); } - template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel) const + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel) noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)))) -> decltype( ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel))) { return ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)); } diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h index cf162100ee46..67818c521b15 100644 --- a/libcxx/include/__ranges/to.h +++ b/libcxx/include/__ranges/to.h @@ -207,7 +207,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) static_assert( !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile"); - auto __to_func = [](_Range&& __range, _Tail&&... __tail) + auto __to_func = [](_Range&& __range, _Tail&&... __tail) static requires requires { // /**/ ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); } @@ -223,7 +223,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) // clang-format off auto __to_func = []::type> - (_Range&& __range, _Tail&& ... __tail) + (_Range&& __range, _Tail&& ... __tail) static requires requires { // /**/ ranges::to<_DeducedExpr>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); } diff --git a/libcxx/include/__ranges/zip_view.h b/libcxx/include/__ranges/zip_view.h index ce00a4e53a48..d3665a149a7c 100644 --- a/libcxx/include/__ranges/zip_view.h +++ b/libcxx/include/__ranges/zip_view.h @@ -489,12 +489,12 @@ namespace views { namespace __zip { struct __fn { - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()() const noexcept { return empty_view>{}; } + _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()() noexcept { return empty_view>{}; } template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ranges&&... __rs) const - noexcept(noexcept(zip_view...>(std::forward<_Ranges>(__rs)...))) - -> decltype(zip_view...>(std::forward<_Ranges>(__rs)...)) { + _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Ranges&&... __rs) noexcept(noexcept(zip_view...>(std::forward<_Ranges>(__rs)...))) + -> decltype(zip_view...>(std::forward<_Ranges>(__rs)...)) { return zip_view...>(std::forward<_Ranges>(__rs)...); } }; -- GitLab From 20e0bacd0560382a31ad0d4ecc7472bd4a99c659 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 19:08:47 -0500 Subject: [PATCH 231/933] [Libomptarget][Fix] Remove duplicate version script for host builds Summary: This causes an error on some linkers and was mistakenly kept in. --- openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index d30680e10431..ccbf7d033fd6 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -31,11 +31,6 @@ else() target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) endif() -if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.${machine} PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") -endif() - # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.${machine} LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") -- GitLab From dc43aa439ecee90a9f51c6c1e46df8be61e0457c Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Fri, 22 Mar 2024 17:38:34 -0700 Subject: [PATCH 232/933] [asan][windows] When compiling with clang-cl or MSVC pass /Zl (that's a little L) (#85874) /Zl is the equivalent of -nodefaultlibs. The idea here is to make sure that the asan runtime doesn't have any defaultlibs directives, which makes it easier to mix an asan runtime built with the dynamic CRT with an application built with the static CRT (or vise-versa). This is part of the overall effort to remove the static asan runtime on windows entirely: https://github.com/llvm/llvm-project/pull/81677 Co-authored-by: Amy Wishnousky --- compiler-rt/lib/asan/CMakeLists.txt | 3 +++ compiler-rt/lib/sanitizer_common/CMakeLists.txt | 2 ++ compiler-rt/lib/ubsan/CMakeLists.txt | 1 + 3 files changed, 6 insertions(+) diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt index f83ae82d4293..601750f72175 100644 --- a/compiler-rt/lib/asan/CMakeLists.txt +++ b/compiler-rt/lib/asan/CMakeLists.txt @@ -85,6 +85,9 @@ SET(ASAN_HEADERS include_directories(..) set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) + +append_list_if(MSVC /Zl ASAN_CFLAGS) + set(ASAN_COMMON_DEFINITIONS ${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION}) append_rtti_flag(OFF ASAN_CFLAGS) diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt index f762524c333a..f2b4ac72ae15 100644 --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -218,6 +218,8 @@ include_directories(..) set(SANITIZER_COMMON_DEFINITIONS HAVE_RPC_XDR_H=${HAVE_RPC_XDR_H}) +# note: L not I, this is nodefaultlibs for msvc +append_list_if(MSVC /Zl SANITIZER_COMMON_CFLAGS) set(SANITIZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) # Too many existing bugs, needs cleanup. diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt index 3f1e12ed9ac6..db0b33f1276e 100644 --- a/compiler-rt/lib/ubsan/CMakeLists.txt +++ b/compiler-rt/lib/ubsan/CMakeLists.txt @@ -41,6 +41,7 @@ set(UBSAN_HEADERS include_directories(..) set(UBSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +append_list_if(MSVC /Zl UBSAN_CFLAGS) append_rtti_flag(OFF UBSAN_CFLAGS) append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CFLAGS) -- GitLab From b723c57f8fdb12ca8eaa6c5ee2afb820aaeb56c5 Mon Sep 17 00:00:00 2001 From: Corentin Jabot Date: Sat, 23 Mar 2024 09:28:33 +0900 Subject: [PATCH 233/933] [Clang] Update the C++ page with papers approved in Tokyo --- clang/www/cxx_status.html | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 1e36b90356c3..c1d95dadbb27 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -163,7 +163,47 @@ C++23, informally referred to as C++26.

P2864R2 Clang 18 - + + + Disallow Binding a Returned Glvalue to a Temporary + P2748R5 + No + + + Clarifying rules for brace elision in aggregate initialization + P3106R1 (DR) + No + + + Attributes for Structured Bindings + P0609R3 + No + + + Module Declarations Shouldn’t be Macros + P3034R1 (DR) + No + + + Trivial infinite loops are not Undefined Behavior + P2809R3 (DR) + No + + + Erroneous behaviour for uninitialized reads + P2795R5 + No + + + = delete("should have a reason"); + P2573R2 + No + + + Variadic friends + P2893R3 + No + -- GitLab From 2f6b1b4b30e3a719b1744baa4cd1ece504998c6e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 18:10:21 -0700 Subject: [PATCH 234/933] [ORC] Add default visibility to required JIT functions (#86322) If you build LLVM with `-DCMAKE_CXX_VISIBILITY_PRESET=hidden` to help reduce binary size, these symbols end up becoming local, and getting stripped. This forces default visibility to override the global setting in that case. Relevant: https://github.com/llvm/llvm-project/issues/62815#issuecomment-1560078260 --- llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 8a4145a6b02a..7529d9cef67e 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -26,11 +26,13 @@ extern "C" { // We put information about the JITed function in this global, which the // debugger reads. Make sure to specify the version statically, because the // debugger checks the version before we can set it during runtime. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0, nullptr, nullptr}; // Debuggers that implement the GDB JIT interface put a special breakpoint in // this function. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { // The noinline and the asm prevent calls to this function from being // optimized out. -- GitLab From 84e20cb0cbc64bb2d8299b30da3ce8e6421c203e Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 16:56:48 -0700 Subject: [PATCH 235/933] [TableGen] Minor cleanup in CodeGenRegBank::getConcatSubRegIndex. NFC Mark a variable const. Capitalize a variable name. I'm going to add HwMode support to this code and wanted to clean it up a bit beforehand. --- llvm/utils/TableGen/CodeGenRegisters.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp index 40af0d3077b2..e851d4c16bf7 100644 --- a/llvm/utils/TableGen/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -1400,11 +1400,11 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( // None exists, synthesize one. std::string Name = Parts.front()->getName(); // Determine whether all parts are contiguous. - bool isContinuous = true; + bool IsContinuous = true; unsigned Size = Parts.front()->Size; unsigned LastOffset = Parts.front()->Offset; unsigned LastSize = Parts.front()->Size; - unsigned UnknownSize = (uint16_t)-1; + const unsigned UnknownSize = (uint16_t)-1; for (unsigned i = 1, e = Parts.size(); i != e; ++i) { Name += '_'; Name += Parts[i]->getName(); @@ -1413,13 +1413,13 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( else Size += Parts[i]->Size; if (LastSize == UnknownSize || Parts[i]->Offset != (LastOffset + LastSize)) - isContinuous = false; + IsContinuous = false; LastOffset = Parts[i]->Offset; LastSize = Parts[i]->Size; } Idx = createSubRegIndex(Name, Parts.front()->getNamespace()); Idx->Size = Size; - Idx->Offset = isContinuous ? Parts.front()->Offset : -1; + Idx->Offset = IsContinuous ? Parts.front()->Offset : -1; Idx->ConcatenationOf.assign(Parts.begin(), Parts.end()); return Idx; } -- GitLab From 76fdb5902fbadbc08c6742156071431d8ad801ea Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 10:37:53 +0800 Subject: [PATCH 236/933] [NewPM][DirectX] Add DirectXPassRegistry.def NFCI (#86242) Prepare migration for dag-isel --- .../Target/DirectX/DirectXPassRegistry.def | 29 +++++++++++++++++++ .../Target/DirectX/DirectXTargetMachine.cpp | 20 ++----------- 2 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 llvm/lib/Target/DirectX/DirectXPassRegistry.def diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def new file mode 100644 index 000000000000..bbf0c254bb69 --- /dev/null +++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def @@ -0,0 +1,29 @@ +//===- DirectXPassRegistry.def - Registry of DirectX passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// DirectX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_ANALYSIS +#define MODULE_ANALYSIS(NAME, CREATE_PASS) +#endif +MODULE_ANALYSIS("dx-shader-flags", dxil::ShaderFlagsAnalysis()) +MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis()) +#undef MODULE_ANALYSIS + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +// TODO: rename to print after NPM switch +MODULE_PASS("print-dx-shader-flags", DXILResourcePrinterPass(dbgs())) +MODULE_PASS("print-dxil-resource", DXILResourcePrinterPass(dbgs())) +#undef MODULE_PASS diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp index 03c825b3977d..bebca0675522 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp @@ -104,24 +104,8 @@ DirectXTargetMachine::~DirectXTargetMachine() {} void DirectXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "print-dxil-resource") { - PM.addPass(DXILResourcePrinterPass(dbgs())); - return true; - } - if (PassName == "print-dx-shader-flags") { - PM.addPass(dxil::ShaderFlagsAnalysisPrinter(dbgs())); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](ModuleAnalysisManager &MAM) { - MAM.registerPass([&] { return DXILResourceAnalysis(); }); - MAM.registerPass([&] { return dxil::ShaderFlagsAnalysis(); }); - }); +#define GET_PASS_REGISTRY "DirectXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" } bool DirectXTargetMachine::addPassesToEmitFile( -- GitLab From 7ac7d418ac2b16fd44789dcf48e2b5d73de3e715 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 11:20:18 +0800 Subject: [PATCH 237/933] [NewPM][NVPTX] Add NVPTXPassRegistry.def NFCI (#86246) Prepare for dag-isel migration. --- llvm/lib/Target/NVPTX/NVPTXPassRegistry.def | 40 +++++++++++++++++++ llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 41 +------------------- 2 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def diff --git a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def new file mode 100644 index 000000000000..6ff15ab6f13c --- /dev/null +++ b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def @@ -0,0 +1,40 @@ +//===- NVPTXPassRegistry.def - Registry of NVPTX passes ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// NVPTX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("generic-to-nvvm", GenericToNVVMPass()) +MODULE_PASS("nvptx-lower-ctor-dtor", NVPTXCtorDtorLoweringPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("nvptx-aa", NVPTXAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("nvvm-intr-range", NVVMIntrRangePass()) +FUNCTION_PASS("nvvm-reflect", NVVMReflectPass()) +#undef FUNCTION_PASS diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 69d4596f7843..78f48652c992 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -227,45 +227,8 @@ void NVPTXTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { void NVPTXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "nvvm-reflect") { - PM.addPass(NVVMReflectPass()); - return true; - } - if (PassName == "nvvm-intr-range") { - PM.addPass(NVVMIntrRangePass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return NVPTXAA(); }); - }); - - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "nvptx-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); - - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "nvptx-lower-ctor-dtor") { - PM.addPass(NVPTXCtorDtorLoweringPass()); - return true; - } - if (PassName == "generic-to-nvvm") { - PM.addPass(GenericToNVVMPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "NVPTXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { -- GitLab From 2aa5bae0c03f1f857d0ae2a881b223c4a521853f Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 12:53:26 +0800 Subject: [PATCH 238/933] [NewPM][BPF] Add BPFPassRegistry.def NFCI (#86241) Prepare migration for dag-isel. --- llvm/lib/Target/BPF/BPFPassRegistry.def | 32 ++++++++++++++++++++++++ llvm/lib/Target/BPF/BPFTargetMachine.cpp | 25 ++++++------------ 2 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 llvm/lib/Target/BPF/BPFPassRegistry.def diff --git a/llvm/lib/Target/BPF/BPFPassRegistry.def b/llvm/lib/Target/BPF/BPFPassRegistry.def new file mode 100644 index 000000000000..73a8ef2f95bf --- /dev/null +++ b/llvm/lib/Target/BPF/BPFPassRegistry.def @@ -0,0 +1,32 @@ +//===- BPFPassRegistry.def - Registry of BPF passes -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// BPF backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("bpf-aspace-simplify", BPFASpaceCastSimplifyPass()) +FUNCTION_PASS("bpf-ir-peephole", BPFIRPeepholePass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "bpf-preserve-static-offset", "BPFPreserveStaticOffsetPass", + [=](bool AllowPartial) { + return BPFPreserveStaticOffsetPass(AllowPartial); + }, + parseBPFPreserveStaticOffsetOptions, "allow-partial") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 5f26bec2e390..a7bed69b0f2a 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -108,25 +108,16 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) { return new BPFPassConfig(*this, PM); } +static Expected parseBPFPreserveStaticOffsetOptions(StringRef Params) { + return PassBuilder::parseSinglePassOption(Params, "allow-partial", + "BPFPreserveStaticOffsetPass"); +} + void BPFTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &FPM, - ArrayRef) { - if (PassName == "bpf-ir-peephole") { - FPM.addPass(BPFIRPeepholePass()); - return true; - } - if (PassName == "bpf-preserve-static-offset") { - FPM.addPass(BPFPreserveStaticOffsetPass(false)); - return true; - } - if (PassName == "bpf-aspace-simplify") { - FPM.addPass(BPFASpaceCastSimplifyPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "BPFPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerPipelineStartEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { FunctionPassManager FPM; -- GitLab From 635ea257eca7c8e95c6ea30ca3816a0b5584ab37 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 13:06:58 +0800 Subject: [PATCH 239/933] [NewPM] Fix BPF build (#86379) Add Passes in dependency list --- llvm/lib/Target/BPF/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index cb21ed03a86c..eade4cacb710 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -54,6 +54,7 @@ add_llvm_target(BPFCodeGen GlobalISel IPO MC + Passes Scalar SelectionDAG Support -- GitLab From 6c1932ffd82e733325180fe13ef46b24ff606eab Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:57:35 +0800 Subject: [PATCH 240/933] [LLVM] Pass APInt by const reference. NFC. (#86278) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=32d6611af69bf4e76373f9bc7d9649650f760e48&stat=instructions:u --- llvm/include/llvm/Analysis/MemoryBuiltins.h | 2 +- llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +- llvm/include/llvm/MC/MCStreamer.h | 2 +- llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 3 ++- llvm/lib/MC/MCStreamer.cpp | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index 37ce1518f00c..da092866de65 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -217,7 +217,7 @@ struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} - static bool known(APInt V) { return V.getBitWidth() > 1; } + static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; /// Evaluate the size and offset of an object pointed to by a Value* diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 4785e93d72d1..574c63552ce0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -883,7 +883,7 @@ public: /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, Step, Step * 2, Step * 3, ...> - SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); + SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal); /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, 1, 2, 3, ...> diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 671511ab4b88..69867620e1bf 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -740,7 +740,7 @@ public: /// Special case of EmitValue that avoids the client having /// to pass in a MCExpr for constant integers. virtual void emitIntValue(uint64_t Value, unsigned Size); - virtual void emitIntValue(APInt Value); + virtual void emitIntValue(const APInt &Value); /// Special case of EmitValue that avoids the client having to pass /// in a MCExpr for constant integers & prints in Hex format for certain diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 8d608f6ac5e4..1b25da8833e4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -7855,7 +7855,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) { //{ (Src & Mask) >> N } | { (Src << N) & Mask } static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, - MachineInstrBuilder Src, APInt Mask) { + MachineInstrBuilder Src, const APInt &Mask) { const LLT Ty = Dst.getLLTTy(*B.getMRI()); MachineInstrBuilder C_N = B.buildConstant(Ty, N); MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index cd6f083243d0..e2c07e7cb997 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2031,7 +2031,8 @@ SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { return getStepVector(DL, ResVT, One); } -SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { +SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, + const APInt &StepVal) { assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); if (ResVT.isScalableVector()) return getNode( diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index d0395770ae8b..176d55aa890b 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -141,7 +141,7 @@ void MCStreamer::emitIntValue(uint64_t Value, unsigned Size) { unsigned Index = IsLittleEndian ? 0 : 8 - Size; emitBytes(StringRef(reinterpret_cast(&Swapped) + Index, Size)); } -void MCStreamer::emitIntValue(APInt Value) { +void MCStreamer::emitIntValue(const APInt &Value) { if (Value.getNumWords() == 1) { emitIntValue(Value.getLimitedValue(), Value.getBitWidth() / 8); return; -- GitLab From 2f1f6b704d83f87be7ea885480caf1c86d8cfaee Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:58:25 +0800 Subject: [PATCH 241/933] [LLVM] Use `std::move` for APInt. NFC. (#86257) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=ba3e326def3a6e5cd6d72ff5a49c74fba18de1df&stat=instructions:u --- llvm/include/llvm/Analysis/InlineCost.h | 3 ++- llvm/include/llvm/Analysis/MemoryBuiltins.h | 6 ++++-- llvm/lib/Analysis/ConstantFolding.cpp | 2 +- llvm/lib/Analysis/InstructionSimplify.cpp | 6 ++++-- llvm/lib/IR/LLVMContextImpl.h | 2 +- llvm/lib/Transforms/Scalar/MergeICmps.cpp | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 3a760e0a85ce..c5978ce54fc1 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -65,7 +65,8 @@ const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize"; // The cost-benefit pair computed by cost-benefit analysis. class CostBenefitPair { public: - CostBenefitPair(APInt Cost, APInt Benefit) : Cost(Cost), Benefit(Benefit) {} + CostBenefitPair(APInt Cost, APInt Benefit) + : Cost(std::move(Cost)), Benefit(std::move(Benefit)) {} const APInt &getCost() const { return Cost; } diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index da092866de65..bb282a1b73d3 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -196,7 +196,8 @@ public: T Offset; SizeOffsetType() = default; - SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {} + SizeOffsetType(T Size, T Offset) + : Size(std::move(Size)), Offset(std::move(Offset)) {} bool knownSize() const { return C::known(Size); } bool knownOffset() const { return C::known(Offset); } @@ -215,7 +216,8 @@ public: /// \p APInts. struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; - SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} + SizeOffsetAPInt(APInt Size, APInt Offset) + : SizeOffsetType(std::move(Size), std::move(Offset)) {} static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index 6139b5be85be..749374a3aa48 100644 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -751,7 +751,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL) { APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0); - return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL); + return ConstantFoldLoadFromConstPtr(C, Ty, std::move(Offset), DL); } Constant *llvm::ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty, diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 7a37ae86c7f3..9ff3faff7990 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6115,7 +6115,8 @@ static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, if (OffsetInt.srem(4) != 0) return nullptr; - Constant *Loaded = ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, OffsetInt, DL); + Constant *Loaded = + ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, std::move(OffsetInt), DL); if (!Loaded) return nullptr; @@ -6983,7 +6984,8 @@ Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp, if (PtrOp == GV) { // Index size may have changed due to address space casts. Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType())); - return ConstantFoldLoadFromConstPtr(GV, LI->getType(), Offset, Q.DL); + return ConstantFoldLoadFromConstPtr(GV, LI->getType(), std::move(Offset), + Q.DL); } return nullptr; diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index 58e0f21244f7..7c67e191348e 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -441,7 +441,7 @@ template <> struct MDNodeKeyImpl { bool IsUnsigned; MDNodeKeyImpl(APInt Value, bool IsUnsigned, MDString *Name) - : Value(Value), Name(Name), IsUnsigned(IsUnsigned) {} + : Value(std::move(Value)), Name(Name), IsUnsigned(IsUnsigned) {} MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name) : Value(APInt(64, Value, !IsUnsigned)), Name(Name), IsUnsigned(IsUnsigned) {} diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp index 1e0906717549..2bd13556c696 100644 --- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp +++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp @@ -74,7 +74,7 @@ namespace { struct BCEAtom { BCEAtom() = default; BCEAtom(GetElementPtrInst *GEP, LoadInst *LoadI, int BaseId, APInt Offset) - : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(Offset) {} + : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(std::move(Offset)) {} BCEAtom(const BCEAtom &) = delete; BCEAtom &operator=(const BCEAtom &) = delete; -- GitLab From ef57977f2aa32661a09fa6538f47ddee0a004d11 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 15:02:27 +0800 Subject: [PATCH 242/933] [NewPM][Hexagon] Add HexagonPassRegistry.def (#86244) Prepare for dag-isel, also migrate some test case --- .../Target/Hexagon/HexagonPassRegistry.def | 21 +++++++++++++++++++ .../Target/Hexagon/HexagonTargetMachine.cpp | 3 +++ .../hexagon_vector_loop_carried_reuse.ll | 2 +- ...n_vector_loop_carried_reuse_commutative.ll | 2 +- ...agon_vector_loop_carried_reuse_constant.ll | 2 +- ...xagon_vector_loop_carried_reuse_invalid.ll | 2 +- .../CodeGen/Hexagon/hvx-loopidiom-memcpy.ll | 1 + .../Hexagon/loop-idiom/hexagon-memmove1.ll | 2 ++ .../Hexagon/loop-idiom/hexagon-memmove2.ll | 2 ++ llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll | 2 +- .../Hexagon/loop-idiom/memmove-rt-check.ll | 1 + .../Hexagon/loop-idiom/nullptr-crash.ll | 1 + .../Hexagon/loop-idiom/pmpy-infinite-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-long-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-shiftconv-fail.ll | 1 + llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll | 2 ++ 16 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 llvm/lib/Target/Hexagon/HexagonPassRegistry.def diff --git a/llvm/lib/Target/Hexagon/HexagonPassRegistry.def b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def new file mode 100644 index 000000000000..4f58ae6193e0 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def @@ -0,0 +1,21 @@ +//===- HexagonPassRegistry.def - Registry of Hexagon passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// Hexagon backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef LOOP_PASS +#define LOOP_PASS(NAME, CREATE_PASS) +#endif +LOOP_PASS("hexagon-loop-idiom", HexagonLoopIdiomRecognitionPass()) +LOOP_PASS("hexagon-vlcr", HexagonVectorLoopCarriedReusePass()) +#undef LOOP_PASS diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 3c346c334d6d..e64d7e52a9aa 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -299,6 +299,9 @@ HexagonTargetMachine::getSubtargetImpl(const Function &F) const { void HexagonTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { +#define GET_PASS_REGISTRY "HexagonPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerLateLoopOptimizationsEPCallback( [=](LoopPassManager &LPM, OptimizationLevel Level) { LPM.addPass(HexagonLoopIdiomRecognitionPass()); diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll index 0771fda02cfb..7ccee1689185 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll index 25afb9f1a137..532f7fd06793 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -march=hexagon -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %v32.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll index 53973423732c..ecfcf53d9133 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK-NOT: %.hexagon.vlcr ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll index b440dba66f67..9872faeb3da0 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll @@ -1,4 +1,4 @@ -; RUN: opt -hexagon-vlcr < %s -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes=hexagon-vlcr -S %s | FileCheck %s ; Test that reuse doesn't occur due to two shufflevectors with different masks. diff --git a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll index ab7bf1b4b0e8..c53e57800edd 100644 --- a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll +++ b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -mtriple=hexagon-- -p hexagon-loop-idiom -disable-memcpy-idiom -S < %s | FileCheck %s ; Make sure we don't convert load/store loops into memcpy if the access type ; is a vector. Using vector instructions is generally better in such cases. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll index c7110263c658..5ace9e6ee486 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll @@ -1,6 +1,8 @@ ; Check for recognizing the "memmove" idiom. ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s ; CHECK: call void @llvm.memmove ; Function Attrs: norecurse nounwind diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll index 234e4f56b5d8..ed56a332f657 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s define void @PR14241(ptr %s, i64 %size) #0 { ; Ensure that we don't form a memcpy for strided loops. Briefly, when we taught diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll index 140c676175ea..e5bcc2b9aebf 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -hexagon-loop-idiom < %s | opt -S -passes='loop(loop-deletion),gvn' +; RUN: opt -mtriple hexagon-- -S -passes='loop(hexagon-loop-idiom,loop-deletion),gvn' ; REQUIRES: asserts ; This tests that the HexagonLoopIdiom pass does not mark LCSSA information diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll index 7a7d1d9b1a86..78f0c9e36b55 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S < %s | FileCheck %s ; Make sure that we generate correct runtime checks. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll index 37e1bb6eb7df..ce02b62911c0 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s +; RUN: opt -p hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s ; REQUIRES: asserts target triple = "hexagon" diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll index 1934ced7e7ae..74c02d63d54d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; CHECK-LABEL: define void @fred ; Check that this test does not crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll index b25010f2a90f..94b0c96c3d51 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; ; The number of nested selects caused the simplification loop to take ; more than the maximum number of iterations. This caused the compiler diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll index e4b2b5a298ed..a00b1d5876ba 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; REQUIRES: asserts ; ; Check for sane output, this used to crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll index 781618e58901..2461e1cfde8d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ +; RUN: | FileCheck %s target triple = "hexagon" -- GitLab From 691b97c884a15a7eac641ddf67c9f2f30fb4e747 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 01:02:08 -0700 Subject: [PATCH 243/933] [ELF] Remove zero-value DT_JMPREL when IPLT is present while PLT isn't The zero-value DT_JMPREL is benign but not needed. This is also code simplification available after https://reviews.llvm.org/D65651 --- lld/ELF/SyntheticSections.cpp | 16 ++------------- .../ELF/aarch64-gnu-ifunc-nonpreemptable.s | 12 +++++------ lld/test/ELF/gnu-ifunc-dyntags.s | 8 +++----- lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s | 8 ++++---- lld/test/ELF/riscv-ifunc-nonpreemptible.s | 20 +++++++++---------- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 248ff6b4a865..10eda17f0cb3 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1285,13 +1285,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { // output section. When this occurs we cannot just use the OutputSection // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to // overlap with the [DT_RELA, DT_RELA + DT_RELASZ). -static uint64_t addPltRelSz() { - size_t size = in.relaPlt->getSize(); - if (in.relaIplt->getParent() == in.relaPlt->getParent() && - in.relaIplt->name == in.relaPlt->name) - size += in.relaIplt->getSize(); - return size; -} +static uint64_t addPltRelSz() { return in.relaPlt->getSize(); } // Add remaining entries to complete .dynamic contents. template @@ -1407,13 +1401,7 @@ DynamicSection::computeContents() { addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT, sizeof(Elf_Relr)); } - // .rel[a].plt section usually consists of two parts, containing plt and - // iplt relocations. It is possible to have only iplt relocations in the - // output. In that case relaPlt is empty and have zero offset, the same offset - // as relaIplt has. And we still want to emit proper dynamic tags for that - // case, so here we always use relaPlt as marker for the beginning of - // .rel[a].plt section. - if (isMain && (in.relaPlt->isNeeded() || in.relaIplt->isNeeded())) { + if (isMain && in.relaPlt->isNeeded()) { addInSec(DT_JMPREL, *in.relaPlt); entries.emplace_back(DT_PLTRELSZ, addPltRelSz()); switch (config->emachine) { diff --git a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s index 4f33dde9d1a9..54a27a8cc0d9 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s +++ b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s @@ -65,14 +65,14 @@ main: # PIE-EMPTY: # PIE-NEXT: : # PIE-NEXT: 10270: adrp x16, 0x30000 -# PIE-NEXT: 10274: ldr x17, [x16, #896] -# PIE-NEXT: 10278: add x16, x16, #896 -# PIE-NEXT: 1027c: br x17 +# PIE-NEXT: ldr x17, [x16, #832] +# PIE-NEXT: add x16, x16, #832 +# PIE-NEXT: br x17 # PIE-RELOC: .rela.dyn { -# PIE-RELOC-NEXT: 0x30380 R_AARCH64_IRELATIVE - 0x10260 +# PIE-RELOC-NEXT: 0x30340 R_AARCH64_IRELATIVE - 0x10260 # PIE-RELOC-NEXT: } # PIE-RELOC: Hex dump of section '.got.plt': -# NO-APPLY: 0x00030380 00000000 00000000 -# APPLY: 0x00030380 60020100 00000000 +# NO-APPLY: 0x00030340 00000000 00000000 +# APPLY: 0x00030340 60020100 00000000 # PIE-RELOC-EMPTY: diff --git a/lld/test/ELF/gnu-ifunc-dyntags.s b/lld/test/ELF/gnu-ifunc-dyntags.s index fd80dc24f2f8..57a17245a3e8 100644 --- a/lld/test/ELF/gnu-ifunc-dyntags.s +++ b/lld/test/ELF/gnu-ifunc-dyntags.s @@ -9,15 +9,13 @@ # CHECK: Name Size VMA # CHECK: .rela.dyn 00000030 0000000000000248 -# CHECK: .got.plt 00000010 00000000000033b0 +# CHECK: .got.plt 00000010 0000000000003370 # TAGS: Tag Type Name/Value # TAGS: 0x0000000000000007 RELA 0x248 # TAGS: 0x0000000000000008 RELASZ 48 (bytes) -# TAGS: 0x0000000000000017 JMPREL 0x0 -# TAGS: 0x0000000000000002 PLTRELSZ 0 (bytes) -# TAGS: 0x0000000000000003 PLTGOT 0x33B0 -# TAGS: 0x0000000000000014 PLTREL RELA +# TAGS-NOT: JMPREL +# TAGS-NOT: PLTREL # TAGS: Relocations [ # TAGS-NEXT: Section {{.*}} .rela.dyn { diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s index a93f3cecb0c6..c9a0381b610a 100644 --- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s +++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s @@ -10,16 +10,16 @@ # RUN: llvm-readelf -x .got2 %t | FileCheck --check-prefix=HEX2 %s # RELOC: .rela.dyn { -# RELOC-NEXT: 0x3024C R_PPC_RELATIVE - 0x101A0 -# RELOC-NEXT: 0x30250 R_PPC_IRELATIVE - 0x10188 +# RELOC-NEXT: 0x3022C R_PPC_RELATIVE - 0x101A0 +# RELOC-NEXT: 0x30230 R_PPC_IRELATIVE - 0x10188 # RELOC-NEXT: } # SYM: 000101a0 0 FUNC GLOBAL DEFAULT {{.*}} func # HEX: Hex dump of section '.got2': -# HEX-NEXT: 0x0003024c 00000000 .... +# HEX-NEXT: 0x0003022c 00000000 .... # HEX2: Hex dump of section '.got2': -# HEX2-NEXT: 0x0003024c 000101a0 .... +# HEX2-NEXT: 0x0003022c 000101a0 .... .section .got2,"aw" .long func diff --git a/lld/test/ELF/riscv-ifunc-nonpreemptible.s b/lld/test/ELF/riscv-ifunc-nonpreemptible.s index 21c607545110..eda5548eef8b 100644 --- a/lld/test/ELF/riscv-ifunc-nonpreemptible.s +++ b/lld/test/ELF/riscv-ifunc-nonpreemptible.s @@ -16,11 +16,11 @@ # RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefix=DIS64 %s # RELOC32: .rela.dyn { -# RELOC32-NEXT: 0x3220 R_RISCV_IRELATIVE - 0x117C +# RELOC32-NEXT: 0x3200 R_RISCV_IRELATIVE - 0x117C # RELOC32-NEXT: } # RELOC32-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC32: 0x00003220 00000000 -# APPLY-RELOC32: 0x00003220 7c110000 +# NO-APPLY-RELOC32: 0x00003200 00000000 +# APPLY-RELOC32: 0x00003200 7c110000 # RELOC32-EMPTY: # SYM32: 0001190 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -30,18 +30,18 @@ # DIS32-NEXT: addi a0, a0, 0x10 # DIS32: Disassembly of section .iplt: # DIS32: : -## 32-bit: &.got.plt[func]-. = 0x3220-0x1190 = 4096*2+144 +## 32-bit: &.got.plt[func]-. = 0x3200-0x1190 = 4096*2+0x70 # DIS32-NEXT: 1190: auipc t3, 0x2 -# DIS32-NEXT: lw t3, 0x90(t3) +# DIS32-NEXT: lw t3, 0x70(t3) # DIS32-NEXT: jalr t1, t3 # DIS32-NEXT: nop # RELOC64: .rela.dyn { -# RELOC64-NEXT: 0x3380 R_RISCV_IRELATIVE - 0x1260 +# RELOC64-NEXT: 0x3340 R_RISCV_IRELATIVE - 0x1260 # RELOC64-NEXT: } # RELOC64-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC64: 0x00003380 00000000 00000000 -# APPLY-RELOC64: 0x00003380 60120000 00000000 +# NO-APPLY-RELOC64: 0x00003340 00000000 00000000 +# APPLY-RELOC64: 0x00003340 60120000 00000000 # RELOC64-EMPTY: # SYM64: 000000000001270 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -51,9 +51,9 @@ # DIS64-NEXT: addi a0, a0, 0xc # DIS64: Disassembly of section .iplt: # DIS64: : -## 64-bit: &.got.plt[func]-. = 0x3380-0x1270 = 4096*2+272 +## 64-bit: &.got.plt[func]-. = 0x3340-0x1270 = 4096*2+0xd0 # DIS64-NEXT: 1270: auipc t3, 0x2 -# DIS64-NEXT: ld t3, 0x110(t3) +# DIS64-NEXT: ld t3, 0xd0(t3) # DIS64-NEXT: jalr t1, t3 # DIS64-NEXT: nop -- GitLab From 579dc7f8441a8044b92bdfa6f0db2f91301c0eed Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:03:45 -0700 Subject: [PATCH 244/933] [clang-forma] Support `PointerAlignment` for pointers to members (#86253) Fixes #85761. --- clang/lib/Format/TokenAnnotator.cpp | 37 +++++++++++------- clang/unittests/Format/FormatTest.cpp | 38 ++++++++++++------- clang/unittests/Format/QualifierFixerTest.cpp | 36 +++++++++--------- 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index a5cafcbacaa5..7757c8ff7639 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -4357,9 +4357,11 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Left.is(tok::kw_auto) && Right.isOneOf(tok::l_paren, tok::l_brace)) return false; + const auto *BeforeLeft = Left.Previous; + // operator co_await(x) - if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && Left.Previous && - Left.Previous->is(tok::kw_operator)) { + if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && BeforeLeft && + BeforeLeft->is(tok::kw_operator)) { return false; } // co_await (x), co_yield (x), co_return (x) @@ -4394,8 +4396,10 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, } if (Left.is(tok::colon)) return Left.isNot(TT_ObjCMethodExpr); - if (Left.is(tok::coloncolon)) - return false; + if (Left.is(tok::coloncolon)) { + return Right.is(tok::star) && Right.is(TT_PointerOrReference) && + Style.PointerAlignment != FormatStyle::PAS_Left; + } if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less)) { if (Style.Language == FormatStyle::LK_TextProto || (Style.Language == FormatStyle::LK_Proto && @@ -4410,8 +4414,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return false; } if (Right.is(tok::ellipsis)) { - return Left.Tok.isLiteral() || (Left.is(tok::identifier) && Left.Previous && - Left.Previous->is(tok::kw_case)); + return Left.Tok.isLiteral() || (Left.is(tok::identifier) && BeforeLeft && + BeforeLeft->is(tok::kw_case)); } if (Left.is(tok::l_square) && Right.is(tok::amp)) return Style.SpacesInSquareBrackets; @@ -4479,8 +4483,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Right.is(tok::l_brace) && Right.is(BK_Block)) return true; // for (auto a = 0, b = 0; const auto& c : {1, 2, 3}) - if (Left.Previous && Left.Previous->isTypeOrIdentifier(IsCpp) && - Right.Next && Right.Next->is(TT_RangeBasedForLoopColon)) { + if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next && + Right.Next->is(TT_RangeBasedForLoopColon)) { return getTokenPointerOrReferenceAlignment(Left) != FormatStyle::PAS_Right; } @@ -4502,12 +4506,17 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, startsWithInitStatement(Line)))) { return false; } - return Left.Previous && !Left.Previous->isOneOf( - tok::l_paren, tok::coloncolon, tok::l_square); + if (!BeforeLeft) + return false; + if (BeforeLeft->is(tok::coloncolon)) { + return Left.is(tok::star) && + Style.PointerAlignment != FormatStyle::PAS_Right; + } + return !BeforeLeft->isOneOf(tok::l_paren, tok::l_square); } // Ensure right pointer alignment with ellipsis e.g. int *...P - if (Left.is(tok::ellipsis) && Left.Previous && - Left.Previous->isPointerOrReference()) { + if (Left.is(tok::ellipsis) && BeforeLeft && + BeforeLeft->isPointerOrReference()) { return Style.PointerAlignment != FormatStyle::PAS_Right; } @@ -4669,13 +4678,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return Style.SpaceBeforeParensOptions.AfterFunctionDefinitionName || spaceRequiredBeforeParens(Right); } - if (!Left.Previous || !Left.Previous->isOneOf(tok::period, tok::arrow)) { + if (!BeforeLeft || !BeforeLeft->isOneOf(tok::period, tok::arrow)) { if (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch)) { return Style.SpaceBeforeParensOptions.AfterControlStatements || spaceRequiredBeforeParens(Right); } if (Left.isOneOf(tok::kw_new, tok::kw_delete)) { - return ((!Line.MightBeFunctionDecl || !Left.Previous) && + return ((!Line.MightBeFunctionDecl || !BeforeLeft) && Style.SpaceBeforeParens != FormatStyle::SBPO_Never) || spaceRequiredBeforeParens(Right); } diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index bea989c8c306..cf8d6ab691d9 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -3621,8 +3621,8 @@ TEST_F(FormatTest, FormatsClasses) { " : public aaaaaaaaaaaaaaaaaaa {};"); verifyFormat("template \n" - "struct Aaaaaaaaaaaaaaaaa\n" - " : Aaaaaaaaaaaaaaaaa {};"); + "struct Aaaaaaaaaaaaaaaaa\n" + " : Aaaaaaaaaaaaaaaaa {};"); verifyFormat("class ::A::B {};"); } @@ -11034,10 +11034,10 @@ TEST_F(FormatTest, UnderstandsBinaryOperators) { } TEST_F(FormatTest, UnderstandsPointersToMembers) { - verifyFormat("int A::*x;"); - verifyFormat("int (S::*func)(void *);"); - verifyFormat("void f() { int (S::*func)(void *); }"); - verifyFormat("typedef bool *(Class::*Member)() const;"); + verifyFormat("int A:: *x;"); + verifyFormat("int (S:: *func)(void *);"); + verifyFormat("void f() { int (S:: *func)(void *); }"); + verifyFormat("typedef bool *(Class:: *Member)() const;"); verifyFormat("void f() {\n" " (a->*f)();\n" " a->*x;\n" @@ -11052,9 +11052,19 @@ TEST_F(FormatTest, UnderstandsPointersToMembers) { verifyFormat( "(aaaaaaaaaa->*bbbbbbb)(\n" " aaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa));"); + FormatStyle Style = getLLVMStyle(); + EXPECT_EQ(Style.PointerAlignment, FormatStyle::PAS_Right); + verifyFormat("typedef bool *(Class:: *Member)() const;", Style); + verifyFormat("void f(int A:: *p) { int A:: *v = &A::B; }", Style); + Style.PointerAlignment = FormatStyle::PAS_Left; - verifyFormat("typedef bool* (Class::*Member)() const;", Style); + verifyFormat("typedef bool* (Class::* Member)() const;", Style); + verifyFormat("void f(int A::* p) { int A::* v = &A::B; }", Style); + + Style.PointerAlignment = FormatStyle::PAS_Middle; + verifyFormat("typedef bool * (Class:: * Member)() const;", Style); + verifyFormat("void f(int A:: * p) { int A:: * v = &A::B; }", Style); } TEST_F(FormatTest, UnderstandsUnaryOperators) { @@ -12386,7 +12396,7 @@ TEST_F(FormatTest, FormatsFunctionTypes) { verifyFormat("int (*func)(void *);"); verifyFormat("void f() { int (*func)(void *); }"); verifyFormat("template \n" - "using MyCallback = void (CallbackClass::*)(SomeObject *Data);"); + "using Callback = void (CallbackClass:: *)(SomeObject *Data);"); verifyGoogleFormat("A;"); verifyGoogleFormat("void* (*a)(int);"); @@ -19149,13 +19159,13 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { "int bbbbbbb = 0;", Alignment); // http://llvm.org/PR68079 - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int (A::*)() &;\n" - "using RRFn = int (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int (A:: *)() &;\n" + "using RRFn = int (A:: *)() &&;", Alignment); - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int *(A::*)() &;\n" - "using RRFn = double (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int *(A:: *)() &;\n" + "using RRFn = double (A:: *)() &&;", Alignment); // PAS_Right diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp index 43476aea6633..792d8f3c3a98 100644 --- a/clang/unittests/Format/QualifierFixerTest.cpp +++ b/clang/unittests/Format/QualifierFixerTest.cpp @@ -305,7 +305,7 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("Foo inline static const;", "Foo inline const static;", Style); verifyFormat("Foo inline static const;", Style); - verifyFormat("Foo::Bar const volatile A::*;", + verifyFormat("Foo::Bar const volatile A:: *;", "volatile const Foo::Bar A::*;", Style); @@ -523,14 +523,15 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("const INTPTR a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("int const S::*a;", "const int S:: *a;", Style); - verifyFormat("int const S::*const a;", "const int S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("float const (C::*p)(int);", "const float (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("int const S:: *a;", "const int S:: *a;", Style); + verifyFormat("int const S:: *const a;", "const int S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("float const (C:: *p)(int);", "const float (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, LeftQualifier) { @@ -830,14 +831,15 @@ TEST_F(QualifierFixerTest, LeftQualifier) { verifyFormat("INTPTR const a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("const int S::*a;", "int const S:: *a;", Style); - verifyFormat("const int S::*const a;", "int const S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("const float (C::*p)(int);", "float const (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("const int S:: *a;", "int const S:: *a;", Style); + verifyFormat("const int S:: *const a;", "int const S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("const float (C:: *p)(int);", "float const (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, ConstVolatileQualifiersOrder) { -- GitLab From 47423e9827abfdcc6b10ce41618965861b0e69a4 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:24:56 -0700 Subject: [PATCH 245/933] [clang-format][NFC] Clean up IsQualifiedPointerOrReference in TokenAnnotator --- clang/lib/Format/TokenAnnotator.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 7757c8ff7639..4c83a7a3a323 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -2753,10 +2753,9 @@ private: } // Heuristically try to determine whether the parentheses contain a type. - auto IsQualifiedPointerOrReference = [this](FormatToken *T) { + auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) { // This is used to handle cases such as x = (foo *const)&y; assert(!T->isTypeName(IsCpp) && "Should have already been checked"); - (void)IsCpp; // Avoid -Wunused-lambda-capture when assertion is disabled. // Strip trailing qualifiers such as const or volatile when checking // whether the parens could be a cast to a pointer/reference type. while (T) { @@ -2789,7 +2788,7 @@ private: !Tok.Previous || Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) || Tok.Previous->isTypeName(IsCpp) || - IsQualifiedPointerOrReference(Tok.Previous); + IsQualifiedPointerOrReference(Tok.Previous, IsCpp); bool ParensCouldEndDecl = Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater); if (ParensAreType && !ParensCouldEndDecl) -- GitLab From f317fd266c184bc8f9f4d7e8e03c043d3406333a Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 15:30:26 +0500 Subject: [PATCH 246/933] Revert "[mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225)" This reverts commit 01b1b0c1f728e2c2639edc654424f50830295989. Breaks following AArch64 SVE buildbots: https://lab.llvm.org/buildbot/#/builders/184/builds/11363 https://lab.llvm.org/buildbot/#/builders/176/builds/9331 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir deleted file mode 100644 index 57d69383c2de..000000000000 --- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir +++ /dev/null @@ -1,60 +0,0 @@ -// DEFINE: %{compile} = mlir-opt %s \ -// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ -// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ -// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t -// DEFINE: %{entry_point} = conv -// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ -// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils - -// RUN: %{compile} | %{run} | FileCheck %s - -func.func @conv() { - // Define input/output tensors - %input_init = tensor.empty() : tensor<1x8x6xi32> - %output_init = tensor.empty() : tensor<1x7x6xi32> - - %five = arith.constant 5 : i32 - %zero = arith.constant 0 : i32 - %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> - %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> - - // Define the filter tensor - %filter = arith.constant dense<[ - [ 1, 2, 3, 4, 5, 6], - [ 11, 12, 13, 14, 15, 16] - ]> : tensor<2x6xi32> - - // static sizes -> dynamic sizes - %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> - %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> - %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> - - // Run the convolution - %res = linalg.depthwise_conv_1d_nwc_wc - ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) - outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> - - // Print the results - // CHECK: SVE: START OF TEST OUTPUT - vector.print str "SVE: START OF TEST OUTPUT\n" - - // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = - // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] - %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> - call @printMemrefI32(%xf) : (tensor<*xi32>) -> () - - // CHECK-NEXT: SVE: END OF TEST OUTPUT - vector.print str "SVE: END OF TEST OUTPUT\n" - - return -} - -module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { - %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op - transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op - transform.yield - } -} - -func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From d7c672834ec863b458af8ca493157e1e31aaf480 Mon Sep 17 00:00:00 2001 From: XChy Date: Sat, 23 Mar 2024 19:00:42 +0800 Subject: [PATCH 247/933] [CodeGen][NFC] Update tests in AArch64/and-sink.ll --- llvm/test/CodeGen/AArch64/and-sink.ll | 60 +++++++++++++++++++++------ 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll index 4d085869de24..f298a55dab72 100644 --- a/llvm/test/CodeGen/AArch64/and-sink.ll +++ b/llvm/test/CodeGen/AArch64/and-sink.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s ; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s ; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s @@ -9,9 +10,18 @@ ; Test that and is sunk into cmp block to form tbz. define dso_local i32 @and_sink1(i32 %a, i1 %c) { ; CHECK-LABEL: and_sink1: -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: tbz w1, #0, .LBB0_3 +; CHECK-NEXT: // %bb.1: // %bb0 +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: tbnz w0, #2, .LBB0_3 +; CHECK-NEXT: // %bb.2: +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB0_3: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink1( ; CHECK-CGP-NOT: and i32 @@ -35,12 +45,30 @@ bb2: ; Test that both 'and' and cmp get sunk to form tbz. define dso_local i32 @and_sink2(i32 %a, i1 %c, i1 %c2) { ; CHECK-LABEL: and_sink2: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:B] -; CHECK: tbz w2, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:C] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, wzr +; CHECK-NEXT: adrp x9, A +; CHECK-NEXT: str wzr, [x9, :lo12:A] +; CHECK-NEXT: tbz w1, #0, .LBB1_5 +; CHECK-NEXT: // %bb.1: // %bb0.preheader +; CHECK-NEXT: adrp x8, B +; CHECK-NEXT: adrp x9, C +; CHECK-NEXT: .LBB1_2: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:B] +; CHECK-NEXT: tbz w2, #0, .LBB1_6 +; CHECK-NEXT: // %bb.3: // %bb1 +; CHECK-NEXT: // in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: str wzr, [x9, :lo12:C] +; CHECK-NEXT: tbnz w0, #2, .LBB1_2 +; CHECK-NEXT: // %bb.4: +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: .LBB1_5: // %common.ret +; CHECK-NEXT: mov w0, w8 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB1_6: +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink2( ; CHECK-CGP-NOT: and i32 @@ -71,10 +99,16 @@ bb3: ; Test that 'and' is not sunk since cbz is a better alternative. define dso_local i32 @and_sink3(i32 %a) { ; CHECK-LABEL: and_sink3: -; CHECK: and [[REG:w[0-9]+]], w0, #0x3 -; CHECK: [[LOOP:.L[A-Z0-9_]+]]: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: cbz [[REG]], [[LOOP]] +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: and w9, w0, #0x3 +; CHECK-NEXT: .LBB2_1: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: cbz w9, .LBB2_1 +; CHECK-NEXT: // %bb.2: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink3( ; CHECK-CGP-NEXT: and i32 -- GitLab From d365a45cb3eaa640b09874fb7984a6a69683c773 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Sat, 23 Mar 2024 15:12:44 +0300 Subject: [PATCH 248/933] [GlobalISel] Introduce G_TRAP, G_DEBUGTRAP, G_UBSANTRAP (#84941) Here we introduce three new GMIR instructions to cover a set of trap intrinsics. The idea behind it is that generic intrinsics shouldn't be used with G_INTRINSIC opcode. These new instructions can match perfectly with existing trap ISD nodes. It allows X86, AArch64, RISCV and Mips to reuse SelectionDAG patterns for selection and avoid manual selection. However AMDGPU is an exception. It selects traps during legalization regardless SelectionDAG or GlobalISel. Since there are not many places where traps are used, this change attempts to clean up all the usages of G_INTRINSIC with trap intrinsics. So, there is no stage when both G_TRAP and G_INTRINSIC_W_SIDE_EFFECTS(@llvm.trap) are allowed. --- llvm/docs/GlobalISel/GenericOpcode.rst | 19 +++++ llvm/docs/LangRef.rst | 6 ++ .../llvm/CodeGen/GlobalISel/IRTranslator.h | 4 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 5 ++ llvm/include/llvm/Support/TargetOpcodes.def | 5 ++ llvm/include/llvm/Target/GenericOpcodes.td | 22 ++++++ .../Target/GlobalISel/SelectionDAGCompat.td | 3 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 47 ++++++++---- llvm/lib/CodeGen/MachineVerifier.cpp | 11 +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 + .../GISel/AArch64InstructionSelector.cpp | 20 ++--- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 23 +++--- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 8 +- llvm/lib/Target/Mips/MipsLegalizerInfo.cpp | 8 -- .../RISCV/GISel/RISCVInstructionSelector.cpp | 27 ------- .../X86/GISel/X86InstructionSelector.cpp | 19 ----- .../GlobalISel/irtranslator-unreachable.ll | 2 +- .../AArch64/GlobalISel/legalize-exceptions.ll | 2 +- .../GlobalISel/legalizer-info-validation.mir | 9 +++ .../AArch64/GlobalISel/select-trap.mir | 2 +- .../AArch64/GlobalISel/uaddo-8-16-bits.mir | 52 ++++++------- .../AMDGPU/GlobalISel/legalize-trap.mir | 4 +- .../trap.mir | 9 ++- .../GlobalISel/instruction-select/trap.mir | 4 +- .../X86/GlobalISel/x86-select-trap.mir | 2 +- llvm/test/CodeGen/X86/isel-traps.ll | 73 +++++++++++++++++++ .../test/MachineVerifier/test_g_ubsantrap.mir | 18 +++++ 27 files changed, 276 insertions(+), 131 deletions(-) rename llvm/test/CodeGen/Mips/GlobalISel/{legalizer => instruction-select}/trap.mir (55%) create mode 100644 llvm/test/CodeGen/X86/isel-traps.ll create mode 100644 llvm/test/MachineVerifier/test_g_ubsantrap.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index ac6217d08e6a..cae2c21b80d7 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -939,6 +939,25 @@ The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`. Unlike SelectionDAG, there is no _VOID variant. Both of these are permitted to have zero, one, or multiple results. +G_TRAP, G_DEBUGTRAP, G_UBSANTRAP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Represents :ref:`llvm.trap `, :ref:`llvm.debugtrap ` +and :ref:`llvm.ubsantrap ` that generate a target dependent +trap instructions. + +.. code-block:: none + + G_TRAP + +.. code-block:: none + + G_DEBUGTRAP + +.. code-block:: none + + G_UBSANTRAP 12 + Variadic Arguments ------------------ diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 8bc1cab01bf0..391fd2d960dd 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -26926,6 +26926,8 @@ Arguments: The argument should be an MDTuple containing any number of MDStrings. +.. _llvm.trap: + '``llvm.trap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26953,6 +26955,8 @@ This intrinsic is lowered to the target dependent trap instruction. If the target does not have a trap instruction, this intrinsic will be lowered to a call of the ``abort()`` function. +.. _llvm.debugtrap: + '``llvm.debugtrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26980,6 +26984,8 @@ This intrinsic is lowered to code which is intended to cause an execution trap with the intention of requesting the attention of a debugger. +.. _llvm.ubsantrap: + '``llvm.ubsantrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 6ae7c1440907..5f28908e998a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -243,6 +243,10 @@ private: bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode); + /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap). + bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder, + unsigned Opcode); + // Translate @llvm.experimental.vector.interleave2 and // @llvm.experimental.vector.deinterleave2 intrinsics for fixed-width vector // types into vector shuffles. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index aaa81342845b..3ba036ae713f 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -2113,6 +2113,11 @@ public: DstMMO, SrcMMO); } + /// Build and insert G_TRAP or G_DEBUGTRAP + MachineInstrBuilder buildTrap(bool Debug = false) { + return buildInstr(Debug ? TargetOpcode::G_DEBUGTRAP : TargetOpcode::G_TRAP); + } + /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width. MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src, const SrcOp &LSB, const SrcOp &Width) { diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 899eaad5842a..5765926d6d93 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -837,6 +837,11 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE) HANDLE_TARGET_OPCODE(G_MEMSET) HANDLE_TARGET_OPCODE(G_BZERO) +/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics +HANDLE_TARGET_OPCODE(G_TRAP) +HANDLE_TARGET_OPCODE(G_DEBUGTRAP) +HANDLE_TARGET_OPCODE(G_UBSANTRAP) + /// Vector reductions HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD) HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 67d405ba96fa..d0f471eb29b6 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1575,6 +1575,28 @@ def G_BZERO : GenericInstruction { let mayStore = true; } +//------------------------------------------------------------------------------ +// Trap intrinsics +//------------------------------------------------------------------------------ +def G_TRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; + let mayStore = true; +} + +def G_DEBUGTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; +} + +def G_UBSANTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i8imm:$kind); + let hasSideEffects = true; +} + //------------------------------------------------------------------------------ // Bitfield extraction. //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index b1f3c500a1b6..4bb9929e2cf8 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -250,6 +250,9 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 757af3b1c4fe..ef95ceec18a1 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1771,6 +1771,32 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, return true; } +bool IRTranslator::translateTrap(const CallInst &CI, + MachineIRBuilder &MIRBuilder, + unsigned Opcode) { + StringRef TrapFuncName = + CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); + if (TrapFuncName.empty()) { + if (Opcode == TargetOpcode::G_UBSANTRAP) { + uint64_t Code = cast(CI.getOperand(0))->getZExtValue(); + MIRBuilder.buildInstr(Opcode, {}, ArrayRef{Code}); + } else { + MIRBuilder.buildInstr(Opcode); + } + return true; + } + + CallLowering::CallLoweringInfo Info; + if (Opcode == TargetOpcode::G_UBSANTRAP) + Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), + CI.getArgOperand(0)->getType(), 0}); + + Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); + Info.CB = &CI; + Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; + return CLI->lowerCall(MIRBuilder, Info); +} + bool IRTranslator::translateVectorInterleave2Intrinsic( const CallInst &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 && @@ -2459,22 +2485,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return true; } case Intrinsic::trap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP); case Intrinsic::debugtrap: - case Intrinsic::ubsantrap: { - StringRef TrapFuncName = - CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); - if (TrapFuncName.empty()) - break; // Use the default handling. - CallLowering::CallLoweringInfo Info; - if (ID == Intrinsic::ubsantrap) { - Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), - CI.getArgOperand(0)->getType(), 0}); - } - Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); - Info.CB = &CI; - Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; - return CLI->lowerCall(MIRBuilder, Info); - } + return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP); + case Intrinsic::ubsantrap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP); case Intrinsic::amdgcn_cs_chain: return translateCallBase(CI, MIRBuilder); case Intrinsic::fptrunc_round: { @@ -3047,7 +3062,7 @@ bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuil } } - MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef()); + MIRBuilder.buildTrap(); return true; } diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 005efe48ac0c..9a1498d4070e 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1867,6 +1867,17 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } + case TargetOpcode::G_UBSANTRAP: { + const MachineOperand &KindOp = MI->getOperand(0); + if (!MI->getOperand(0).isImm()) { + report("Crash kind must be an immediate", &KindOp, 0); + break; + } + int64_t Kind = MI->getOperand(0).getImm(); + if (!isInt<8>(Kind)) + report("Crash kind must be 8 bit wide", &KindOp, 0); + break; + } case TargetOpcode::G_VECREDUCE_SEQ_FADD: case TargetOpcode::G_VECREDUCE_SEQ_FMUL: { LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index b4b975cce007..b1f514f75207 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -8436,6 +8436,9 @@ def ubsan_trap_xform : SDNodeXFormgetTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32); }]>; +def gi_ubsan_trap_xform : GICustomOperandRenderer<"renderUbsanTrap">, + GISDNodeXFormEquiv; + def ubsan_trap_imm : TImmLeaf(Imm); }], ubsan_trap_xform>; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 677dd0b502b9..a8f2c45279e6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -479,6 +479,8 @@ private: int OpIdx = -1) const; void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx = -1) const; + void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx = -1) const; void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, @@ -6159,16 +6161,6 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects( constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); break; } - case Intrinsic::trap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000); - break; - case Intrinsic::ubsantrap: - MIB.buildInstr(AArch64::BRK, {}, {}) - .addImm(I.getOperand(1).getImm() | ('U' << 8)); - break; case Intrinsic::aarch64_neon_ld1x2: { LLT Ty = MRI.getType(I.getOperand(0).getReg()); unsigned Opc = 0; @@ -7663,6 +7655,14 @@ void AArch64InstructionSelector::renderLogicalImm64( MIB.addImm(Enc); } +void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 && + "Expected G_UBSANTRAP"); + MIB.addImm(MI.getOperand(0).getImm() | ('U' << 8)); +} + void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 90872516dd6d..e55d1de01b4f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2030,6 +2030,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET}) .lower(); + getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom(); + getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE, G_INDEXED_LOAD, G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD, G_INDEXED_STORE}) @@ -2134,6 +2136,10 @@ bool AMDGPULegalizerInfo::legalizeCustom( return legalizeGetFPEnv(MI, MRI, B); case TargetOpcode::G_SET_FPENV: return legalizeSetFPEnv(MI, MRI, B); + case TargetOpcode::G_TRAP: + return legalizeTrap(MI, MRI, B); + case TargetOpcode::G_DEBUGTRAP: + return legalizeDebugTrap(MI, MRI, B); default: return false; } @@ -2925,7 +2931,7 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue( // functions that use local objects. However, if these dead functions are // not eliminated, we don't want a compile time error. Just emit a warning // and a trap, since there should be no callable path here. - B.buildIntrinsic(Intrinsic::trap, ArrayRef()); + B.buildTrap(); B.buildUndef(DstReg); MI.eraseFromParent(); return true; @@ -6618,9 +6624,9 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper, } // TODO: Move to selection -bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI, - MachineRegisterInfo &MRI, - MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { if (!ST.isTrapHandlerEnabled() || ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) return legalizeTrapEndpgm(MI, MRI, B); @@ -6726,8 +6732,9 @@ bool AMDGPULegalizerInfo::legalizeTrapHsa( return true; } -bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic( - MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { // Is non-HSA path or trap-handler disabled? Then, report a warning // accordingly if (!ST.isTrapHandlerEnabled() || @@ -7270,10 +7277,6 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, case Intrinsic::amdgcn_struct_buffer_atomic_fadd_v2bf16: case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd_v2bf16: return legalizeBufferAtomic(MI, B, IntrID); - case Intrinsic::trap: - return legalizeTrapIntrinsic(MI, MRI, B); - case Intrinsic::debugtrap: - return legalizeDebugTrapIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_rsq_clamp: return legalizeRsqClampIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_ds_fadd: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index 9661646fffc9..e5ba84a74a0f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -226,16 +226,16 @@ public: bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const; - bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; - bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override; diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp index 3307e840a2af..8468dd6a2211 100644 --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp @@ -508,16 +508,8 @@ bool MipsLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const { MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; const MipsSubtarget &ST = MI.getMF()->getSubtarget(); - const MipsInstrInfo &TII = *ST.getInstrInfo(); - const MipsRegisterInfo &TRI = *ST.getRegisterInfo(); - const RegisterBankInfo &RBI = *ST.getRegBankInfo(); switch (cast(MI).getIntrinsicID()) { - case Intrinsic::trap: { - MachineInstr *Trap = MIRBuilder.buildInstr(Mips::TRAP); - MI.eraseFromParent(); - return constrainSelectedInstRegOperands(*Trap, TII, TRI, RBI); - } case Intrinsic::vacopy: { MachinePointerInfo MPO; LLT PtrTy = LLT::pointer(0, 32); diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index 5738f86e7e9f..3103992a86c0 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -77,8 +77,6 @@ private: MachineRegisterInfo &MRI) const; bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const; - bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB, - MachineRegisterInfo &MRI) const; void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const; bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB, @@ -686,8 +684,6 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) { return selectSelect(MI, MIB, MRI); case TargetOpcode::G_FCMP: return selectFPCompare(MI, MIB, MRI); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWithSideEffects(MI, MIB, MRI); case TargetOpcode::G_FENCE: { AtomicOrdering FenceOrdering = static_cast(MI.getOperand(0).getImm()); @@ -1255,29 +1251,6 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI, return true; } -bool RISCVInstructionSelector::selectIntrinsicWithSideEffects( - MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const { - assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "Unexpected opcode"); - // Find the intrinsic ID. - unsigned IntrinID = cast(MI).getIntrinsicID(); - - // Select the instruction. - switch (IntrinID) { - default: - return false; - case Intrinsic::trap: - MIB.buildInstr(RISCV::UNIMP, {}, {}); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(RISCV::EBREAK, {}, {}); - break; - } - - MI.eraseFromParent(); - return true; -} - void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const { diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp index 8e0f61a85566..9be3812300af 100644 --- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp +++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp @@ -119,8 +119,6 @@ private: MachineFunction &MF) const; bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; - bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, - MachineFunction &MF) const; // emit insert subreg instruction and insert it before MachineInstr &I bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, @@ -434,8 +432,6 @@ bool X86InstructionSelector::select(MachineInstr &I) { return selectMulDivRem(I, MRI, MF); case TargetOpcode::G_SELECT: return selectSelect(I, MRI, MF); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWSideEffects(I, MRI, MF); } return false; @@ -1834,21 +1830,6 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I, return true; } -bool X86InstructionSelector::selectIntrinsicWSideEffects( - MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { - - assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "unexpected instruction"); - - if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) - return false; - - BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); - - I.eraseFromParent(); - return true; -} - InstructionSelector * llvm::createX86InstructionSelector(const X86TargetMachine &TM, X86Subtarget &Subtarget, diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll index fe9427d2678a..edae903fae84 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll @@ -6,7 +6,7 @@ declare void @llvm.trap() define void @unreachable() { ; CHECK-LABEL: name: unreachable ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP unreachable ret void } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll index 5662de4cbdca..f7550ceb2799 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll @@ -48,7 +48,7 @@ define void @bar() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: $x0 = COPY [[LOAD]](p0) ; CHECK-NEXT: BL @_Unwind_Resume, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP %exn.slot = alloca ptr %ehselector.slot = alloca i32 %1 = invoke i32 @foo(i32 42) to label %continue unwind label %cleanup diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index c9e5f8924f8a..ac3c47c8001d 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -752,6 +752,15 @@ # DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_DEBUGTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_UBSANTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir index ad66fa5623e3..25ecce4dd92b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir @@ -26,7 +26,7 @@ body: | ; CHECK-LABEL: name: foo ; CHECK: BRK 1 ; CHECK: RET_ReallyLR - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP RET_ReallyLR ... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir index f4366fb7888e..b242c68e3b07 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir @@ -26,7 +26,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -48,7 +48,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s8) @@ -80,7 +80,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -102,7 +102,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -134,7 +134,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -165,7 +165,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -206,7 +206,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -228,7 +228,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ANYEXT %6(s16) @@ -261,7 +261,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -284,7 +284,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -317,7 +317,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -340,7 +340,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -377,7 +377,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -410,7 +410,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -512,7 +512,7 @@ body: | ; CHECK-NEXT: bb.2: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.3: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s16) @@ -544,7 +544,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %6(s16) @@ -577,7 +577,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -601,7 +601,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -634,7 +634,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -658,7 +658,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -692,7 +692,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -717,7 +717,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %10:_(s32) = G_ZEXT %8(s8) @@ -783,7 +783,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDO]](s16) @@ -804,7 +804,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %6:_(s32) = G_ANYEXT %4(s16) @@ -839,7 +839,7 @@ body: | ; CHECK-NEXT: RET_ReallyLR implicit $w0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP bb.1: successors: %bb.2(0x7ffff800), %bb.3(0x00000800) liveins: $w0, $w1 @@ -860,6 +860,6 @@ body: | RET_ReallyLR implicit $w0 bb.3: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir index b4bc64812b53..305eca792cfb 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir @@ -24,7 +24,7 @@ body: | bb.0: %0:_(s8) = G_CONSTANT i8 0 %1:_(p1) = G_CONSTANT i64 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.1: G_STORE %0, %1 :: (store 1, addrspace 1) @@ -55,7 +55,7 @@ body: | ; GCN-NEXT: S_ENDPGM 0 bb.0: %0:_(s8) = G_CONSTANT i8 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP %1:_(p1) = G_CONSTANT i64 0 bb.1: diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir similarity index 55% rename from llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir rename to llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir index 64388933fda8..dc99ce8d7a09 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 +# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 --- | declare void @llvm.trap() @@ -9,12 +9,15 @@ --- name: f alignment: 4 +legalized: true +regBankSelected: true +tracksRegLiveness: true body: | bb.1 (%ir-block.0): ; MIPS32-LABEL: name: f ; MIPS32: TRAP - ; MIPS32: RetRA - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; MIPS32-NEXT: RetRA + G_TRAP RetRA ... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir index 11789a030e6f..5f52030fc170 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir @@ -14,7 +14,7 @@ body: | ; CHECK-LABEL: name: test_trap ; CHECK: UNIMP ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP PseudoRET ... @@ -28,7 +28,7 @@ body: | ; CHECK-LABEL: name: test_debugtrap ; CHECK: EBREAK ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.debugtrap) + G_DEBUGTRAP PseudoRET ... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir index ea548c296dca..20b8b671ac5a 100644 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir +++ b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir @@ -23,6 +23,6 @@ body: | bb.1 (%ir-block.0): ; CHECK-LABEL: name: trap ; CHECK: TRAP - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/X86/isel-traps.ll b/llvm/test/CodeGen/X86/isel-traps.ll new file mode 100644 index 000000000000..c207387166a6 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-traps.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X64 +; RUN: llc < %s -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X86 + +declare void @llvm.trap() + +define void @test_trap() { +; ALL-LABEL: test_trap: +; ALL: # %bb.0: +; ALL-NEXT: ud2 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.trap() + ret void +} + +define void @test_debugtrap() { +; ALL-LABEL: test_debugtrap: +; ALL: # %bb.0: +; ALL-NEXT: int3 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.debugtrap() + ret void +} + +define void @test_ubsantrap() { +; ALL-LABEL: test_ubsantrap: +; ALL: # %bb.0: +; ALL-NEXT: ud1l 12(%eax), %eax +; ALL-NEXT: ret{{[l|q]}} + call void @llvm.ubsantrap(i8 12) + ret void +} + +define void @test_ubsantrap_custom() nounwind { +; X64-LABEL: test_ubsantrap_custom: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: movl $42, %edi +; X64-NEXT: callq guide@PLT +; X64-NEXT: popq %rax +; X64-NEXT: retq +; +; GISEL-X64-LABEL: test_ubsantrap_custom: +; GISEL-X64: # %bb.0: +; GISEL-X64-NEXT: pushq %rax +; GISEL-X64-NEXT: movl $42, %edi +; GISEL-X64-NEXT: callq guide +; GISEL-X64-NEXT: popq %rax +; GISEL-X64-NEXT: retq +; +; X86-LABEL: test_ubsantrap_custom: +; X86: # %bb.0: +; X86-NEXT: subl $12, %esp +; X86-NEXT: movl $42, (%esp) +; X86-NEXT: calll guide +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl +; +; GISEL-X86-LABEL: test_ubsantrap_custom: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: subl $12, %esp +; GISEL-X86-NEXT: movl $42, %eax +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: calll guide +; GISEL-X86-NEXT: addl $12, %esp +; GISEL-X86-NEXT: retl + call void @llvm.ubsantrap(i8 42) "trap-func-name"="guide" + ret void +} diff --git a/llvm/test/MachineVerifier/test_g_ubsantrap.mir b/llvm/test/MachineVerifier/test_g_ubsantrap.mir new file mode 100644 index 000000000000..d2b219d8650a --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_ubsantrap.mir @@ -0,0 +1,18 @@ +# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target + +--- +name: test_ubsantrap +tracksRegLiveness: true +liveins: +body: | + bb.0: + + ; CHECK: Crash kind must be 8 bit wide + G_UBSANTRAP 4096 + + ; CHECK: Crash kind must be an immediate + %5:_(s32) = IMPLICIT_DEF + G_UBSANTRAP %5 + +... -- GitLab From f886dfed3ae6cf70827cedc8d8aefde6250a239b Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 13:54:35 +0100 Subject: [PATCH 249/933] [libc++] Don't push and pop extensions diagnostics when using clang modules (#85917) Clang modules take a significant compile time hit when pushing and popping diagnostics. Since all the headers are marked as system headers in the modulemap, we can simply disable this pushing and popping when building with clang modules. --- libcxx/include/__config | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libcxx/include/__config b/libcxx/include/__config index 132cace3d5b2..af44af203e7c 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -838,21 +838,33 @@ typedef __char32_t char32_t; # define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions") #endif +// Clang modules take a significant compile time hit when pushing and popping diagnostics. +// Since all the headers are marked as system headers in the modulemap, we can simply disable this +// pushing and popping when building with clang modules. +# if !__has_feature(modules) +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ + _LIBCPP_DIAGNOSTIC_PUSH \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS _LIBCPP_DIAGNOSTIC_POP +# else +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS +# endif + // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect. // clang-format off -# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") \ +# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std { \ inline namespace _LIBCPP_ABI_NAMESPACE { -# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP +# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_POP_EXTENSION_DIAGNOSTICS # define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD \ inline namespace __fs { namespace filesystem { -- GitLab From 57146daeaaf366050dc913db910fcc2995a3e06d Mon Sep 17 00:00:00 2001 From: Harvin Iriawan <25712785+harviniriawan@users.noreply.github.com> Date: Sat, 23 Mar 2024 12:56:25 +0000 Subject: [PATCH 250/933] [CodeGen] Update for scalable MemoryType in MMO (#70452) Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable TypeSize, the MemoryType created becomes a scalable_vector. 2 MMOs that have scalable memory access can then use the updated BasicAA that understands scalable LocationSize. Original Patch by Harvin Iriawan Co-authored-by: David Green --- llvm/include/llvm/Analysis/MemoryLocation.h | 7 --- llvm/include/llvm/CodeGen/MachineFunction.h | 5 +- llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 34 +++++++++---- llvm/lib/CodeGen/MachineInstr.cpp | 37 ++++++++++---- llvm/lib/CodeGen/MachineOperand.cpp | 13 ++--- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++++++------- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 22 ++++---- .../SelectionDAGAddressAnalysis.cpp | 2 - .../SelectionDAG/SelectionDAGBuilder.cpp | 3 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 5 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++-- llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-- .../alloca-load-store-scalable-array.ll | 36 ++++++------- .../alloca-load-store-scalable-struct.ll | 12 ++--- .../rvv/alloca-load-store-scalable-array.ll | 12 ++--- .../rvv/alloca-load-store-scalable-struct.ll | 8 +-- .../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +-- 17 files changed, 165 insertions(+), 118 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 830eed5d60ee..7d896c44f467 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -297,13 +297,6 @@ public: return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags); } - // Return the exact size if the exact size is known at compiletime, - // otherwise return LocationSize::beforeOrAfterPointer(). - static LocationSize getSizeOrUnknown(const TypeSize &T) { - return T.isScalable() ? LocationSize::beforeOrAfterPointer() - : LocationSize::precise(T.getFixedValue()); - } - MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {} explicit MemoryLocation(const Value *Ptr, LocationSize Size, diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index dfbf7a1e7aae..c2bff2794493 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -1058,8 +1058,9 @@ public: int64_t Offset, LocationSize Size) { return getMachineMemOperand( MMO, Offset, - !Size.hasValue() || Size.isScalable() - ? LLT() + !Size.hasValue() ? LLT() + : Size.isScalable() + ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue()) : LLT::scalar(8 * Size.getValue().getKnownMinValue())); } MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index 9fc8ecd60b03..fb9656c09ca3 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -128,14 +128,14 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1, // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: - if (PtrDiff >= 0 && Size1.hasValue()) { + if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // ========PtrDiff========> IsAlias = !((int64_t)Size1.getValue() <= PtrDiff); return true; } - if (PtrDiff < 0 && Size2.hasValue()) { + if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // =====(-PtrDiff)====> @@ -248,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) || + (MUC1.NumBytes.isScalable() && MUC1.Offset != 0)) + return true; + + const bool BothNotScalable = + !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable(); + // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; - if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) + if (BothNotScalable && + GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) return IsAlias; // The following all rely on MMO0 and MMO1 being valid. @@ -267,12 +277,18 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; - if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0, - MUC0.MMO->getAAInfo()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, - MUC1.MMO->getAAInfo()))) + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); + + if (AA->isNoAlias( + MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()), + MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo()))) return false; } diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index fe2f9ccd33a3..8102bb971ba6 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1306,6 +1306,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, LocationSize WidthB = MMOb->getSize(); bool KnownWidthA = WidthA.hasValue(); bool KnownWidthB = WidthB.hasValue(); + bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable(); const Value *ValA = MMOa->getValue(); const Value *ValB = MMOb->getValue(); @@ -1321,12 +1322,14 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, SameVal = true; } - if (SameVal) { + if (SameVal && BothMMONonScalable) { if (!KnownWidthA || !KnownWidthB) return true; int64_t MaxOffset = std::max(OffsetA, OffsetB); - LocationSize LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB; - return (MinOffset + (int)LowWidth.getValue() > MaxOffset); + int64_t LowWidth = (MinOffset == OffsetA) + ? WidthA.getValue().getKnownMinValue() + : WidthB.getValue().getKnownMinValue(); + return (MinOffset + LowWidth > MaxOffset); } if (!AA) @@ -1338,15 +1341,29 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, assert((OffsetA >= 0) && "Negative MachineMemOperand offset"); assert((OffsetB >= 0) && "Negative MachineMemOperand offset"); - int64_t OverlapA = KnownWidthA ? WidthA.getValue() + OffsetA - MinOffset - : MemoryLocation::UnknownSize; - int64_t OverlapB = KnownWidthB ? WidthB.getValue() + OffsetB - MinOffset - : MemoryLocation::UnknownSize; + // If Scalable Location Size has non-zero offset, Width + Offset does not work + // at the moment + if ((WidthA.isScalable() && OffsetA > 0) || + (WidthB.isScalable() && OffsetB > 0)) + return true; + + int64_t OverlapA = + KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset + : MemoryLocation::UnknownSize; + int64_t OverlapB = + KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset + : MemoryLocation::UnknownSize; + + LocationSize LocA = (WidthA.isScalable() || !KnownWidthA) + ? WidthA + : LocationSize::precise(OverlapA); + LocationSize LocB = (WidthB.isScalable() || !KnownWidthB) + ? WidthB + : LocationSize::precise(OverlapB); return !AA->isNoAlias( - MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), - MemoryLocation(ValB, OverlapB, - UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); + MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), + MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); } bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index 937ca539513a..ace05902d5df 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -1107,12 +1107,13 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F, const MDNode *Ranges, SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) - : MachineMemOperand(ptrinfo, F, - !TS.hasValue() || TS.isScalable() - ? LLT() - : LLT::scalar(8 * TS.getValue().getKnownMinValue()), - BaseAlignment, AAInfo, Ranges, SSID, Ordering, - FailureOrdering) {} + : MachineMemOperand( + ptrinfo, F, + !TS.hasValue() ? LLT() + : TS.isScalable() + ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue()) + : LLT::scalar(8 * TS.getValue().getKnownMinValue()), + BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {} void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) { // The Value and Offset may differ due to CSE. But the flags and size diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index dcd0310734ad..e27a8bb8fdac 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -24200,7 +24200,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) { // TODO: Use "BaseIndexOffset" to make this more effective. SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL); - LocationSize StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); + LocationSize StoreSize = LocationSize::precise(VT.getStoreSize()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO; if (Offset.isScalable()) { @@ -27845,14 +27845,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { : (LSN->getAddressingMode() == ISD::PRE_DEC) ? -1 * C->getSExtValue() : 0; - LocationSize Size = - MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize()); - return {LSN->isVolatile(), - LSN->isAtomic(), - LSN->getBasePtr(), - Offset /*base offset*/, - Size, - LSN->getMemOperand()}; + TypeSize Size = LSN->getMemoryVT().getStoreSize(); + return {LSN->isVolatile(), LSN->isAtomic(), + LSN->getBasePtr(), Offset /*base offset*/, + LocationSize::precise(Size), LSN->getMemOperand()}; } if (const auto *LN = cast(N)) return {false /*isVolatile*/, @@ -27894,6 +27890,13 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() && + MUC0.Offset != 0) || + (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() && + MUC1.Offset != 0)) + return true; // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; @@ -27924,18 +27927,22 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Align OrigAlignment1 = MUC1.MMO->getBaseAlign(); LocationSize Size0 = MUC0.NumBytes; LocationSize Size1 = MUC1.NumBytes; + if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 && - Size0.hasValue() && Size1.hasValue() && Size0 == Size1 && - OrigAlignment0 > Size0.getValue() && - SrcValOffset0 % Size0.getValue() == 0 && - SrcValOffset1 % Size1.getValue() == 0) { + Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() && + !Size1.isScalable() && Size0 == Size1 && + OrigAlignment0 > Size0.getValue().getKnownMinValue() && + SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 && + SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) { int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value(); int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value(); // There is no overlap between these relatively aligned accesses of // similar size. Return no alias. - if ((OffAlign0 + (int64_t)Size0.getValue()) <= OffAlign1 || - (OffAlign1 + (int64_t)Size1.getValue()) <= OffAlign0) + if ((OffAlign0 + static_cast( + Size0.getValue().getKnownMinValue())) <= OffAlign1 || + (OffAlign1 + static_cast( + Size1.getValue().getKnownMinValue())) <= OffAlign0) return false; } @@ -27952,12 +27959,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Size0.hasValue() && Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); if (AA->isNoAlias( - MemoryLocation(MUC0.MMO->getValue(), Overlap0, + MemoryLocation(MUC0.MMO->getValue(), Loc0, UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, + MemoryLocation(MUC1.MMO->getValue(), Loc1, UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes()))) return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e2c07e7cb997..0ab5142ab816 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8406,9 +8406,7 @@ SDValue SelectionDAG::getMemIntrinsicNode( EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags, LocationSize Size, const AAMDNodes &AAInfo) { - if (Size.hasValue() && MemVT.isScalableVector()) - Size = LocationSize::beforeOrAfterPointer(); - else if (Size.hasValue() && !Size.getValue()) + if (Size.hasValue() && !Size.getValue()) Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); @@ -8571,7 +8569,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8692,8 +8690,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); MachineFunction &MF = getMachineFunction(); - LocationSize Size = - MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); + LocationSize Size = LocationSize::precise(Val.getValueType().getStoreSize()); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); return getStore(Chain, dl, Val, Ptr, MMO); @@ -8746,8 +8743,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -8841,7 +8838,7 @@ SDValue SelectionDAG::getLoadVP( if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8994,8 +8991,8 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, IsCompressing); } @@ -11734,10 +11731,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely. - // TODO: Make MachineMemOperands aware of scalable vectors. assert( (!MMO->getType().isValid() || - memvt.getStoreSize().getKnownMinValue() <= MMO->getSize().getValue()) && + TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) && "Size mismatch!"); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 9670c3ac8430..f2ab88851b78 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -106,8 +106,6 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0, int64_t PtrDiff; if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { // If the size of memory access is unknown, do not use it to analysis. - // One example of unknown size memory access is to load/store scalable - // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 84df98b8a613..ae6bd7e938e8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4962,7 +4962,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(AS), MachineMemOperand::MOLoad, - LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), Ranges); + LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), + Ranges); if (!UniformBase) { Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 02943b8a4ab1..d0c5e6b99e9e 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -2687,10 +2687,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth( return false; // The maximum vscale is 16 under AArch64, return the maximal extent for the // vector. - Width = WidthN.isScalable() - ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector / - AArch64::SVEBitsPerBlock - : WidthN.getKnownMinValue(); + Width = LocationSize::precise(WidthN); BaseOps.push_back(BaseOp); return true; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5a2fb0239e0a..5214595485ca 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -10364,9 +10364,15 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op, RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Load->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); SDValue NewLoad = - DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), - Load->getMemOperand()); + DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO); SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL); } @@ -10424,9 +10430,17 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op, const auto [MinVLMAX, MaxVLMAX] = RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && - getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) + getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Store->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(), - Store->getMemOperand()); + MMO); + } SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll index 58299696e78f..d4d803a91cfa 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll @@ -3,7 +3,7 @@ define void @UphPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_p8to15 = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1 ; CHECK: RET_ReallyLR @@ -17,7 +17,7 @@ entry: define void @UpaPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1 ; CHECK: RET_ReallyLR @@ -31,7 +31,7 @@ entry: define void @UplPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_3b = COPY %0 ; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1 ; CHECK: RET_ReallyLR diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll index 9a4e01a29ecb..7244ac949ab8 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll @@ -14,12 +14,12 @@ define void @array_1D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret @@ -81,18 +81,18 @@ define void @array_2D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #5, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #4, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #5, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #4, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] -; CHECK-NEXT: st1d { z5.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z4.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #5, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #4, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #5, mul vl] +; CHECK-NEXT: st1d { z3.d }, p0, [sp, #4, mul vl] +; CHECK-NEXT: st1d { z5.d }, p0, [sp, #3, mul vl] +; CHECK-NEXT: st1d { z4.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #6 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll index 7292d52aaf47..f03a6f018d34 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll @@ -13,12 +13,12 @@ define void @test(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll index 1d025a2f776f..1fe91c721f4d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll @@ -18,15 +18,15 @@ define void @test(ptr %addr) { ; CHECK-NEXT: add a2, a0, a1 ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: slli a2, a1, 1 -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v9, (a3) +; CHECK-NEXT: vl1re64.v v9, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v10, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v9, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v9, (a2) -; CHECK-NEXT: add a1, a0, a1 -; CHECK-NEXT: vs1r.v v8, (a1) -; CHECK-NEXT: vs1r.v v10, (a0) +; CHECK-NEXT: vs1r.v v10, (a2) +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: csrrs a0, vlenb, zero ; CHECK-NEXT: slli a0, a0, 2 ; CHECK-NEXT: add sp, sp, a0 diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll index 64031f8a9359..a9a680d54d58 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll @@ -16,13 +16,13 @@ define @test(ptr %addr, i64 %vl) { ; CHECK-NEXT: sub sp, sp, a2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb ; CHECK-NEXT: csrrs a2, vlenb, zero -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v8, (a3) +; CHECK-NEXT: vl1re64.v v8, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v8, (a2) -; CHECK-NEXT: vs1r.v v9, (a0) +; CHECK-NEXT: vs1r.v v9, (a2) ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll index 0544204cce79..52bd15742ef4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll @@ -16,8 +16,8 @@ define void @vpmerge_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer @@ -37,8 +37,8 @@ define void @vpselect_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer -- GitLab From b68e2eba0bc8dd70b88f4271831139ee9b6ed25c Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 15:28:22 +0100 Subject: [PATCH 251/933] [libc++] Vectorize mismatch (#73255) ``` --------------------------------------------------- Benchmark old new --------------------------------------------------- bm_mismatch/1 0.835 ns 2.37 ns bm_mismatch/2 1.44 ns 2.60 ns bm_mismatch/3 2.06 ns 2.83 ns bm_mismatch/4 2.60 ns 3.29 ns bm_mismatch/5 3.15 ns 3.77 ns bm_mismatch/6 3.82 ns 4.17 ns bm_mismatch/7 4.29 ns 4.52 ns bm_mismatch/8 4.78 ns 4.86 ns bm_mismatch/16 9.06 ns 7.54 ns bm_mismatch/64 31.7 ns 19.1 ns bm_mismatch/512 249 ns 8.16 ns bm_mismatch/4096 1956 ns 44.2 ns bm_mismatch/32768 15498 ns 501 ns bm_mismatch/262144 123965 ns 4479 ns bm_mismatch/1048576 495668 ns 21306 ns bm_mismatch/1 0.710 ns 2.12 ns bm_mismatch/2 1.03 ns 2.66 ns bm_mismatch/3 1.29 ns 3.56 ns bm_mismatch/4 1.68 ns 4.29 ns bm_mismatch/5 1.96 ns 5.18 ns bm_mismatch/6 2.59 ns 5.91 ns bm_mismatch/7 2.86 ns 6.63 ns bm_mismatch/8 3.19 ns 7.33 ns bm_mismatch/16 5.48 ns 13.0 ns bm_mismatch/64 16.6 ns 4.06 ns bm_mismatch/512 130 ns 13.8 ns bm_mismatch/4096 985 ns 93.8 ns bm_mismatch/32768 7846 ns 1002 ns bm_mismatch/262144 63217 ns 10637 ns bm_mismatch/1048576 251782 ns 42471 ns bm_mismatch/1 0.716 ns 1.91 ns bm_mismatch/2 1.21 ns 2.49 ns bm_mismatch/3 1.38 ns 3.46 ns bm_mismatch/4 1.71 ns 4.04 ns bm_mismatch/5 2.00 ns 4.98 ns bm_mismatch/6 2.43 ns 5.67 ns bm_mismatch/7 3.05 ns 6.38 ns bm_mismatch/8 3.22 ns 7.09 ns bm_mismatch/16 5.18 ns 12.8 ns bm_mismatch/64 16.6 ns 5.28 ns bm_mismatch/512 129 ns 25.2 ns bm_mismatch/4096 1009 ns 201 ns bm_mismatch/32768 7776 ns 2144 ns bm_mismatch/262144 62371 ns 20551 ns bm_mismatch/1048576 254750 ns 90097 ns ``` --- libcxx/benchmarks/CMakeLists.txt | 1 + .../benchmarks/algorithms/mismatch.bench.cpp | 31 +++ libcxx/docs/ReleaseNotes/19.rst | 2 + libcxx/include/CMakeLists.txt | 1 + libcxx/include/__algorithm/mismatch.h | 82 ++++++- libcxx/include/__algorithm/simd_utils.h | 123 ++++++++++ libcxx/include/__bit/bit_cast.h | 9 + libcxx/include/__bit/countr.h | 13 +- libcxx/include/libcxx.imp | 1 + libcxx/include/module.modulemap | 6 +- .../mismatch/mismatch.pass.cpp | 214 +++++++++++++----- .../mismatch/mismatch_pred.pass.cpp | 119 ---------- 12 files changed, 413 insertions(+), 189 deletions(-) create mode 100644 libcxx/benchmarks/algorithms/mismatch.bench.cpp create mode 100644 libcxx/include/__algorithm/simd_utils.h delete mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt index 3dec6faea13a..387e013afeb6 100644 --- a/libcxx/benchmarks/CMakeLists.txt +++ b/libcxx/benchmarks/CMakeLists.txt @@ -183,6 +183,7 @@ set(BENCHMARK_TESTS algorithms/make_heap_then_sort_heap.bench.cpp algorithms/min.bench.cpp algorithms/min_max_element.bench.cpp + algorithms/mismatch.bench.cpp algorithms/pop_heap.bench.cpp algorithms/pstl.stable_sort.bench.cpp algorithms/push_heap.bench.cpp diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp new file mode 100644 index 000000000000..9274932a764c --- /dev/null +++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +// TODO: Look into benchmarking aligned and unaligned memory explicitly +// (currently things happen to be aligned because they are malloced that way) +template +static void bm_mismatch(benchmark::State& state) { + std::vector vec1(state.range(), '1'); + std::vector vec2(state.range(), '1'); + std::mt19937_64 rng(std::random_device{}()); + + vec1.back() = '2'; + for (auto _ : state) { + benchmark::DoNotOptimize(vec1); + benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin())); + } +} +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); + +BENCHMARK_MAIN(); diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index cac42f9c3c3f..dd39c1bbbc78 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -51,6 +51,8 @@ Improvements and New Features - The performance of growing ``std::vector`` has been improved for trivially relocatable types. - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector::iterator``\s, resulting in a performance increase of up to 1400x. +- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance + improvements. Deprecations and Removals ------------------------- diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 6ed8d21d98a1..982b85e4e2d6 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -217,6 +217,7 @@ set(files __algorithm/shift_right.h __algorithm/shuffle.h __algorithm/sift_down.h + __algorithm/simd_utils.h __algorithm/sort.h __algorithm/sort_heap.h __algorithm/stable_partition.h diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index d345b6048a7e..4eb693a1f2e9 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -11,23 +11,93 @@ #define _LIBCPP___ALGORITHM_MISMATCH_H #include <__algorithm/comp.h> +#include <__algorithm/simd_utils.h> +#include <__algorithm/unwrap_iter.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__functional/identity.h> +#include <__type_traits/invoke.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_equality_comparable.h> +#include <__type_traits/operation_traits.h> +#include <__utility/move.h> #include <__utility/pair.h> +#include <__utility/unreachable.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + while (__first1 != __last1) { + if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) + break; + ++__first1; + ++__first2; + } + return std::make_pair(std::move(__first1), std::move(__first2)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#if _LIBCPP_VECTORIZE_ALGORITHMS + +template ::value && __desugars_to<__equal_tag, _Pred, _Tp, _Tp>::value && + __is_identity<_Proj1>::value && __is_identity<_Proj2>::value, + int> = 0> +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*> +__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + constexpr size_t __unroll_count = 4; + constexpr size_t __vec_size = __native_vector_size<_Tp>; + using __vec = __simd_vector<_Tp, __vec_size>; + if (!__libcpp_is_constant_evaluated()) { + while (static_cast(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] { + __vec __lhs[__unroll_count]; + __vec __rhs[__unroll_count]; + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + __lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size); + __rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size); + } + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) { + auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res); + return {__first1 + __offset, __first2 + __offset}; + } + } + + __first1 += __unroll_count * __vec_size; + __first2 += __unroll_count * __vec_size; + } + } + // TODO: Consider vectorizing the tail + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#endif // _LIBCPP_VECTORIZE_ALGORITHMS + template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) { - for (; __first1 != __last1; ++__first1, (void)++__first2) - if (!__pred(*__first1, *__first2)) - break; - return pair<_InputIterator1, _InputIterator2>(__first1, __first2); + __identity __proj; + auto __res = std::__mismatch( + std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj); + return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second)); } template @@ -59,4 +129,6 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP___ALGORITHM_MISMATCH_H diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h new file mode 100644 index 000000000000..1aedb3db010f --- /dev/null +++ b/libcxx/include/__algorithm/simd_utils.h @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H +#define _LIBCPP___ALGORITHM_SIMD_UTILS_H + +#include <__bit/bit_cast.h> +#include <__bit/countr.h> +#include <__config> +#include <__type_traits/is_arithmetic.h> +#include <__type_traits/is_same.h> +#include <__utility/integer_sequence.h> +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// TODO: Find out how altivec changes things and allow vectorizations there too. +#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__) +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1 +#else +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) +# define _LIBCPP_VECTORIZE_ALGORITHMS 1 +#else +# define _LIBCPP_VECTORIZE_ALGORITHMS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +_LIBCPP_BEGIN_NAMESPACE_STD + +// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance +// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms +// as far as benchmarks are concerned. +# if defined(__AVX__) +template +inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); +# elif defined(__SSE__) || defined(__ARM_NEON__) +template +inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); +# elif defined(__MMX__) +template +inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); +# else +template +inline constexpr size_t __native_vector_size = 1; +# endif + +template +using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; + +template +inline constexpr size_t __simd_vector_size_v = []() -> size_t { + static_assert(_False, "Not a vector!"); +}(); + +template +inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; + +template +_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { + return _Tp{}; +} + +template +using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); + +// This isn't inlined without always_inline when loading chars. +template +_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept { + return [=](index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { + return _VecT{__ptr[_Indices]...}; + }(make_index_sequence<__simd_vector_size_v<_VecT>>{}); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { + return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { + using __mask_vec = __simd_vector; + + // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 + auto __impl = [&](_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { + return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))); + }; + + if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { + return __impl(uint8_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { + return __impl(uint16_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { + return __impl(uint32_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { + return __impl(uint64_t{}); + } else { + static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); + return 0; + } +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { + return std::__find_first_set(~__vec); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H diff --git a/libcxx/include/__bit/bit_cast.h b/libcxx/include/__bit/bit_cast.h index f20b39ae748b..6298810f3733 100644 --- a/libcxx/include/__bit/bit_cast.h +++ b/libcxx/include/__bit/bit_cast.h @@ -19,6 +19,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD +#ifndef _LIBCPP_CXX03_LANG + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept { + return __builtin_bit_cast(_ToType, __from); +} + +#endif // _LIBCPP_CXX03_LANG + #if _LIBCPP_STD_VER >= 20 template diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h index 0cc679f87a99..b6b3ac52ca4e 100644 --- a/libcxx/include/__bit/countr.h +++ b/libcxx/include/__bit/countr.h @@ -35,10 +35,8 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ct return __builtin_ctzll(__x); } -#if _LIBCPP_STD_VER >= 20 - -template <__libcpp_unsigned_integer _Tp> -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT { if (__t == 0) return numeric_limits<_Tp>::digits; @@ -59,6 +57,13 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n } } +#if _LIBCPP_STD_VER >= 20 + +template <__libcpp_unsigned_integer _Tp> +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { + return std::__countr_zero(__t); +} + template <__libcpp_unsigned_integer _Tp> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept { return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits; diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index 77b7befd44f5..56ea58262828 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -217,6 +217,7 @@ { include: [ "<__algorithm/shift_right.h>", "private", "", "public" ] }, { include: [ "<__algorithm/shuffle.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sift_down.h>", "private", "", "public" ] }, + { include: [ "<__algorithm/simd_utils.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort_heap.h>", "private", "", "public" ] }, { include: [ "<__algorithm/stable_partition.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index f36a47cef009..03d18775631e 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -697,7 +697,10 @@ module std_private_algorithm_minmax [system export * } module std_private_algorithm_minmax_element [system] { header "__algorithm/minmax_element.h" } -module std_private_algorithm_mismatch [system] { header "__algorithm/mismatch.h" } +module std_private_algorithm_mismatch [system] { + header "__algorithm/mismatch.h" + export std_private_algorithm_simd_utils +} module std_private_algorithm_move [system] { header "__algorithm/move.h" } module std_private_algorithm_move_backward [system] { header "__algorithm/move_backward.h" } module std_private_algorithm_next_permutation [system] { header "__algorithm/next_permutation.h" } @@ -1048,6 +1051,7 @@ module std_private_algorithm_sort [system header "__algorithm/sort.h" export std_private_debug_utils_strict_weak_ordering_check } +module std_private_algorithm_simd_utils [system] { header "__algorithm/simd_utils.h" } module std_private_algorithm_sort_heap [system] { header "__algorithm/sort_heap.h" } module std_private_algorithm_stable_partition [system] { header "__algorithm/stable_partition.h" } module std_private_algorithm_stable_sort [system] { header "__algorithm/stable_sort.h" } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp index cc588c095ccf..e7f3994d977d 100644 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp @@ -16,79 +16,173 @@ // template // constexpr pair // constexpr after c++17 // mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2); // C++14 +// +// template Pred> +// requires CopyConstructible +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); +// +// template +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 + +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=50000000 +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-ops-limit): -fconstexpr-ops-limit=100000000 #include +#include #include +#include #include "test_macros.h" #include "test_iterators.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic)); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic)); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic)); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic)); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic))); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic))); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } +#include "type_algorithms.h" + +template +TEST_CONSTEXPR_CXX20 void check(Container1 lhs, Container2 rhs, size_t offset) { + if (lhs.size() == rhs.size()) { + assert(std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + } + +#if TEST_STD_VER >= 14 + assert( + std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data()), Iter(rhs.data() + rhs.size())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + Iter(rhs.data() + rhs.size()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); #endif +} -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib)) - == (std::pair(II(ia+3), II(ib+3)))); - - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - -#if TEST_STD_VER > 11 // We have the four iteration version - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+sb)) - == (std::pair(II(ia+3), II(ib+3)))); +struct NonTrivial { + int i_; + + TEST_CONSTEXPR_CXX20 NonTrivial(int i) : i_(i) {} + TEST_CONSTEXPR_CXX20 NonTrivial(NonTrivial&& other) : i_(other.i_) { other.i_ = 0; } + + TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivial& lhs, const NonTrivial& rhs) { return lhs.i_ == rhs.i_; } +}; + +struct ModTwoComp { + TEST_CONSTEXPR_CXX20 bool operator()(int lhs, int rhs) { return lhs % 2 == rhs % 2; } +}; + +template +TEST_CONSTEXPR_CXX20 bool test() { + { // empty ranges + std::array lhs = {}; + std::array rhs = {}; + check(lhs, rhs, 0); + } + + { // same range without mismatch + std::array lhs = {0, 1, 2, 3, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 8); + } + + { // same range with mismatch + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 3); + } + + { // second range is smaller + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1}; + check(lhs, rhs, 2); + } + + { // first range is smaller + std::array lhs = {0, 1}; + std::array rhs = {0, 1, 2, 2, 0, 1, 2, 3}; + check(lhs, rhs, 2); + } + + { // use a custom comparator + std::array lhs = {0, 2, 3, 4}; + std::array rhs = {0, 0, 4, 4}; + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#if TEST_STD_VER >= 14 + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), rhs.data() + rhs.size(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#endif + } - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib+sb)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); + return true; +} +struct Test { + template + TEST_CONSTEXPR_CXX20 void operator()() { + test(); + } +}; + +TEST_CONSTEXPR_CXX20 bool test() { + types::for_each(types::cpp17_input_iterator_list(), Test()); + + { // use a non-integer type to also test the general case - all elements match + std::array lhs = {1, 2, 3, 4, 5, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 8); + } + + { // use a non-integer type to also test the general case - not all elements match + std::array lhs = {1, 2, 3, 4, 7, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 4); + } + + return true; +} - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+2)) - == (std::pair(II(ia+2), II(ib+2)))); +int main(int, char**) { + test(); +#if TEST_STD_VER >= 20 + static_assert(test()); #endif -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif + { // check with a lot of elements to test the vectorization optimization + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + } return 0; } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp deleted file mode 100644 index bda4ec7ba5ed..000000000000 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template Pred> -// requires CopyConstructible -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); -// -// template -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 - -#include -#include -#include - -#include "test_macros.h" -#include "test_iterators.h" -#include "counting_predicates.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool eq(int a, int b) { return a == b; } - -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), eq); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic), eq); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), eq); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic), eq); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic)), eq); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic)), eq); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } -#endif - - -#if TEST_STD_VER > 11 -#define HAS_FOUR_ITERATOR_VERSION -#endif - -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - typedef std::equal_to EQ; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - binary_counting_predicate bcp((EQ())); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), std::ref(bcp)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < sa); - bcp.reset(); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib + sb), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), std::ref(bcp)) - == (std::pair(II(ia+3), II(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < std::min(sa, sb)); -#endif - - assert(std::mismatch(ia, ia + sa, ib, EQ()) == - (std::pair(ia+3,ib+3))); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(ia, ia + sa, ib, ib + sb, EQ()) == - (std::pair(ia+3,ib+3))); - assert(std::mismatch(ia, ia + sa, ib, ib + 2, EQ()) == - (std::pair(ia+2,ib+2))); -#endif - -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif - - return 0; -} -- GitLab From 3f5e649ff64a92a732027ef76e33a1c8b1722d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 16:46:32 +0100 Subject: [PATCH 252/933] [Libomptarget] Fix linking to LLVM dylib (#86397) Use `LINK_COMPONENTS` parameter of `add_llvm_library` rather than passing LLVM components directly to `target_link_libraries`, in order to ensure that LLVM dylib is linked correctly when used. Otherwise, CMake insists on linking to static libraries that aren't present on distributions doing pure dylib installs, such as Gentoo. This fixes a regression introduced in dcbddc25250158469c5635ad2ae4095faef53dfd. --- .../plugins-nextgen/CMakeLists.txt | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index c19fd80592d6..dbd82ac94517 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -14,41 +14,40 @@ set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) function(add_target_library target_name lib_name) - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) - add_llvm_library(${target_name} SHARED + LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + AggressiveInstCombine + Analysis + BinaryFormat + BitReader + BitWriter + CodeGen + Core + Extensions + InstCombine + Instrumentation + IPO + IRReader + Linker + MC + Object + Passes + Remarks + ScalarOpts + Support + Target + TargetParser + TransformUtils + Vectorize + NO_INSTALL_RPATH BUILDTREE_ONLY ) llvm_update_compile_flags(${target_name}) target_link_libraries(${target_name} PRIVATE - PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) + PluginCommon ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) target_compile_definitions(${target_name} PRIVATE -- GitLab From d7ce6b4d96c8879f38ba4cb5fdb1cc09d5b129e5 Mon Sep 17 00:00:00 2001 From: Mike Rice Date: Sat, 23 Mar 2024 09:53:29 -0700 Subject: [PATCH 253/933] [clang-tidy] Fix result check after overwriteChangedFiles() (#86360) If any return from overwriteChangedFiles is true some fixes were not applied. --- clang-tools-extra/clang-tidy/ClangTidy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp index 40ac6918faf4..b877ea06dc05 100644 --- a/clang-tools-extra/clang-tidy/ClangTidy.cpp +++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp @@ -233,7 +233,7 @@ public: if (!tooling::applyAllReplacements(Replacements.get(), Rewrite)) { llvm::errs() << "Can't apply replacements for file " << File << "\n"; } - AnyNotWritten &= Rewrite.overwriteChangedFiles(); + AnyNotWritten |= Rewrite.overwriteChangedFiles(); } if (AnyNotWritten) { -- GitLab From 87c7f4a12b2a1b723a78d760761ee473f52c4cee Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:03:09 -0700 Subject: [PATCH 254/933] [MC] Remove unnecessary reversal of relocations. NFC Commit f44db24e1fd948c75c87aea017646f16553d3361 (2015) enabled this simplication. --- llvm/lib/MC/ELFObjectWriter.cpp | 9 +-------- .../lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 5 ++--- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index 3c4d3ab9a508..a6fb6b5c1a4e 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -939,18 +939,11 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec) { std::vector &Relocs = OWriter.Relocations[&Sec]; - // We record relocations by pushing to the end of a vector. Reverse the vector - // to get the relocations in the order they were created. - // In most cases that is not important, but it can be for special sections - // (.eh_frame) or specific relocations (TLS optimizations on SystemZ). - std::reverse(Relocs.begin(), Relocs.end()); - // Sort the relocation entries. MIPS needs this. OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { - const ELFRelocationEntry &Entry = Relocs[e - i - 1]; + for (const ELFRelocationEntry &Entry : Relocs) { unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; if (is64Bit()) { diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp index 181b82f14bfe..4d6a00c14a35 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp @@ -498,10 +498,9 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm, assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed"); - // Overwrite the original vector with the sorted elements. The caller expects - // them in reverse order. + // Overwrite the original vector with the sorted elements. unsigned CopyTo = 0; - for (const auto &R : reverse(Sorted)) + for (const auto &R : Sorted) Relocs[CopyTo++] = R.R; } -- GitLab From 3a63f737e29a2382c8ec26c0d360bd77c01a27a1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:15:47 -0700 Subject: [PATCH 255/933] [MC] Refactor writeRelocations. NFC MIPS is different and should better off use separate code. --- llvm/lib/MC/ELFObjectWriter.cpp | 60 ++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index a6fb6b5c1a4e..f4c6cbc8dd44 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -943,40 +943,28 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (const ELFRelocationEntry &Entry : Relocs) { - unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; - - if (is64Bit()) { - write(Entry.Offset); - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { - write(uint32_t(Index)); - + if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + write(uint32_t(Symidx)); write(OWriter.TargetObjectWriter->getRSsym(Entry.Type)); write(OWriter.TargetObjectWriter->getRType3(Entry.Type)); write(OWriter.TargetObjectWriter->getRType2(Entry.Type)); write(OWriter.TargetObjectWriter->getRType(Entry.Type)); + if (Rela) + write(Entry.Addend); } else { - struct ELF::Elf64_Rela ERE64; - ERE64.setSymbolAndType(Index, Entry.Type); - write(ERE64.r_info); - } - if (Rela) - write(Entry.Addend); - } else { - write(uint32_t(Entry.Offset)); - - struct ELF::Elf32_Rela ERE32; - ERE32.setSymbolAndType(Index, Entry.Type); - write(ERE32.r_info); - - if (Rela) - write(uint32_t(Entry.Addend)); - - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE32; + ERE32.setSymbolAndType(Symidx, Entry.Type); + write(ERE32.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); if (uint32_t RType = OWriter.TargetObjectWriter->getRType2(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); @@ -984,13 +972,31 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, if (uint32_t RType = OWriter.TargetObjectWriter->getRType3(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); } } } + return; + } + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + ELF::Elf64_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(Entry.Addend); + } else { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); + } } } -- GitLab From 39c8e87717fbc611b9e84f62edf656608ae52e5c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 23 Mar 2024 18:43:14 +0100 Subject: [PATCH 256/933] [VPlan] Move recording of Inst->VPValue to VPRecipeBuilder (NFCI). (#84464) Instead of keeping a mapping of Inst->VPValues (of their corresponding recipes) in VPlan's Value2VPValue mapping, keep it in VPRecipeBuilder instead. After recently replacing the last user of this mapping after initial construction, this mapping is only needed for recipe construction (to map IR operands to VPValue operands). By moving the mapping, VPlan's VPValue tracking can be simplified and limited only to live-ins. It also allows removing disableValue2VPValue and associated machinery & asserts. PR: https://github.com/llvm/llvm-project/pull/84464 --- .../Transforms/Vectorize/LoopVectorize.cpp | 47 +++++++------------ .../Transforms/Vectorize/VPRecipeBuilder.h | 35 +++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 25 ++-------- .../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++--- 4 files changed, 46 insertions(+), 76 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2163930b02c1..28748142ae92 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7898,6 +7898,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, } } +iterator_range>> +VPRecipeBuilder::mapToVPValues(User::op_range Operands) { + std::function Fn = [this](Value *Op) { + if (auto *I = dyn_cast(Op)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(Op); + }; + return map_range(Operands, Fn); +} + VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { assert(is_contained(predecessors(Dst), Src) && "Invalid edge"); @@ -7922,7 +7934,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { if (OrigLoop->isLoopExiting(Src)) return EdgeMaskCache[Edge] = SrcMask; - VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition()); + VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan); assert(EdgeMask && "No Edge Mask found for condition"); if (BI->getSuccessor(0) != Dst) @@ -8383,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I, BlockInMask = getBlockInMask(I->getParent()); } - auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()), + auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()), IsUniform, BlockInMask); return Recipe; } @@ -8399,10 +8411,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, if (Phi->getParent() != OrigLoop->getHeader()) return tryToBlend(Phi, Operands); - // Always record recipes for header phis. Later first-order recurrence phis - // can have earlier phis as incoming values. - recordRecipeOf(Phi); - if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range))) return Recipe; @@ -8427,14 +8435,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } - // Record the incoming value from the backedge, so we can add the incoming - // value from the backedge after all recipes have been created. - auto *Inc = cast( - Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); - auto RecipeIter = Ingredient2Recipe.find(Inc); - if (RecipeIter == Ingredient2Recipe.end()) - recordRecipeOf(Inc); - PhisToFix.push_back(PhiRecipe); return PhiRecipe; } @@ -8522,7 +8522,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the // original exit block. static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, - VPlan &Plan) { + VPRecipeBuilder &Builder, VPlan &Plan) { BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock(); BasicBlock *ExitingBB = OrigLoop->getExitingBlock(); // Only handle single-exit loops with unique exit blocks for now. @@ -8533,7 +8533,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, for (PHINode &ExitPhi : ExitBB->phis()) { Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB); - VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue); + VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan); Plan.addLiveOut(&ExitPhi, V); } } @@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!getDecisionAndClampRange(applyIG, Range)) continue; InterleaveGroups.insert(IG); - for (unsigned i = 0; i < IG->getFactor(); i++) - if (Instruction *Member = IG->getMember(i)) - RecipeBuilder.recordRecipeOf(Member); }; // --------------------------------------------------------------------------- @@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Operands.push_back(Plan->getVPValueOrAddLiveIn( Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); } else { - auto OpRange = Plan->mapToVPValues(Instr->operands()); + auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands()); Operands = {OpRange.begin(), OpRange.end()}; } @@ -8662,10 +8659,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB); if (!Recipe) Recipe = RecipeBuilder.handleReplication(Instr, Range); - for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); - } RecipeBuilder.setRecipe(Instr, Recipe); if (isa(Recipe)) { @@ -8697,7 +8690,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // and there is nothing to fix from vector loop; phis should have incoming // from scalar loop only. } else - addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan); + addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan); assert(isa(Plan->getVectorLoopRegion()) && !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && @@ -8765,10 +8758,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV); } - // From this point onwards, VPlan-to-VPlan transformations may change the plan - // in ways that accessing values using original IR values is incorrect. - Plan->disableValue2VPValue(); - VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) { return Legal->blockNeedsPredication(BB); }); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 29a395c35731..fbb70f333685 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -52,9 +52,8 @@ class VPRecipeBuilder { EdgeMaskCacheTy EdgeMaskCache; BlockMaskCacheTy BlockMaskCache; - // VPlan-VPlan transformations support: Hold a mapping from ingredients to - // their recipe. To save on memory, only do so for selected ingredients, - // marked by having a nullptr entry in this map. + // VPlan construction support: Hold a mapping from ingredients to + // their recipe. DenseMap Ingredient2Recipe; /// Cross-iteration reduction & first-order recurrence phis for which we need @@ -117,13 +116,10 @@ public: ArrayRef Operands, VFRange &Range, VPBasicBlock *VPBB); - /// Set the recipe created for given ingredient. This operation is a no-op for - /// ingredients that were not marked using a nullptr entry in the map. + /// Set the recipe created for given ingredient. void setRecipe(Instruction *I, VPRecipeBase *R) { - if (!Ingredient2Recipe.count(I)) - return; - assert(Ingredient2Recipe[I] == nullptr && - "Recipe already set for ingredient"); + assert(!Ingredient2Recipe.contains(I) && + "Cannot reset recipe for instruction."); Ingredient2Recipe[I] = R; } @@ -146,14 +142,6 @@ public: /// between SRC and DST. VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const; - /// Mark given ingredient for recording its recipe once one is created for - /// it. - void recordRecipeOf(Instruction *I) { - assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) && - "Recipe already set for ingredient"); - Ingredient2Recipe[I] = nullptr; - } - /// Return the recipe created for given ingredient. VPRecipeBase *getRecipe(Instruction *I) { assert(Ingredient2Recipe.count(I) && @@ -171,6 +159,19 @@ public: /// Add the incoming values from the backedge to reduction & first-order /// recurrence cross-iteration phis. void fixHeaderPhis(); + + /// Returns a range mapping the values of the range \p Operands to their + /// corresponding VPValues. + iterator_range>> + mapToVPValues(User::op_range Operands); + + VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) { + if (auto *I = dyn_cast(V)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(V); + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d77c7554d50e..22173954f7ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2872,10 +2872,6 @@ class VPlan { /// definitions are VPValues that hold a pointer to their underlying IR. SmallVector VPLiveInsToFree; - /// Indicates whether it is safe use the Value2VPValue mapping or if the - /// mapping cannot be used any longer, because it is stale. - bool Value2VPValueEnabled = true; - /// Values used outside the plan. MapVector LiveOuts; @@ -2954,10 +2950,6 @@ public: /// Returns VF * UF of the vector loop region. VPValue &getVFxUF() { return VFxUF; } - /// Mark the plan to indicate that using Value2VPValue is not safe any - /// longer, because it may be stale. - void disableValue2VPValue() { Value2VPValueEnabled = false; } - void addVF(ElementCount VF) { VFs.insert(VF); } void setVF(ElementCount VF) { @@ -2987,8 +2979,7 @@ public: void setName(const Twine &newName) { Name = newName.str(); } void addVPValue(Value *V, VPValue *VPV) { - assert((Value2VPValueEnabled || VPV->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(VPV->isLiveIn() && "VPV must be a live-in."); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); Value2VPValue[V] = VPV; @@ -2998,8 +2989,8 @@ public: VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(Value2VPValue[V]->isLiveIn() && + "Only live-ins should be in mapping"); return Value2VPValue[V]; } @@ -3030,16 +3021,6 @@ public: LLVM_DUMP_METHOD void dump() const; #endif - /// Returns a range mapping the values the range \p Operands to their - /// corresponding VPValues. - iterator_range>> - mapToVPValues(User::op_range Operands) { - std::function Fn = [this](Value *Op) { - return getVPValueOrAddLiveIn(Op); - }; - return map_range(Operands, Fn); - } - /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { return cast(getEntry()->getSingleSuccessor()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a91ccefe4b6d..00cec1148528 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -48,15 +48,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes( VPRecipeBase *NewRecipe = nullptr; if (auto *VPPhi = dyn_cast(&Ingredient)) { auto *Phi = cast(VPPhi->getUnderlyingValue()); - if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) { - VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); - VPValue *Step = - vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); - } else { - Plan->addVPValue(Phi, VPPhi); + const auto *II = GetIntOrFpInductionDescriptor(Phi); + if (!II) continue; - } + + VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); + VPValue *Step = + vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); } else { assert(isa(&Ingredient) && "only VPInstructions expected here"); -- GitLab From e14c6fa31a303312d7561aa0c7219d687f3f2c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 20:26:20 +0100 Subject: [PATCH 257/933] [clang] [cmake] Add cmake module dir before using GetDarwinLinkerVersion (#86386) Move the code adding top-level cmake/Modules directory to CMAKE_MODULE_PATH prior to including `GetDarwinLinkerVersion`, in order to fix standalone builds. Fixes a regression introduced by 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b. --- clang/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index ee783d52e4a4..284b2af24dda 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -13,6 +13,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(CLANG_BUILT_STANDALONE TRUE) endif() +# Make sure that our source directory is on the current cmake module path so that +# we can include cmake files from this directory. +list(INSERT CMAKE_MODULE_PATH 0 + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" + "${LLVM_COMMON_CMAKE_UTILS}/Modules" + ) + # Must go below project(..) include(GNUInstallDirs) include(GetDarwinLinkerVersion) @@ -141,13 +148,6 @@ if(CLANG_BUILT_STANDALONE) endif() # LLVM_INCLUDE_TESTS endif() # standalone -# Make sure that our source directory is on the current cmake module path so that -# we can include cmake files from this directory. -list(INSERT CMAKE_MODULE_PATH 0 - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" - "${LLVM_COMMON_CMAKE_UTILS}/Modules" - ) - # This allows disabling clang's XML dependency even if LLVM finds libxml2. # By default, clang depends on libxml2 if LLVM does. option(CLANG_ENABLE_LIBXML2 "Whether libclang may depend on libxml2" -- GitLab From a91cd53de34fcde469d5d02e84554a06798ad2b1 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 16:08:31 -0700 Subject: [PATCH 258/933] [BOLT][NFC] Refactor BAT metadata data structures Hide the implementations of `FuncHashes` and `BBHashMap` classes, getting rid of `at` accessors that could throw an exception. Test Plan: NFC Reviewers: ayermolo, maksfb, dcci, rafaelauler Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86353 --- .../bolt/Profile/BoltAddressTranslation.h | 117 +++++++++++++++--- bolt/lib/Profile/BoltAddressTranslation.cpp | 85 ++++++------- 2 files changed, 137 insertions(+), 65 deletions(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index d583ce0b76a2..f8c35f8066f7 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -115,17 +115,6 @@ public: /// Save function and basic block hashes used for metadata dump. void saveMetadata(BinaryContext &BC); - /// Returns BB hash by function output address (after BOLT) and basic block - /// input offset. - size_t getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - - /// Returns BF hash by function output address (after BOLT). - size_t getBFHash(uint64_t OutputAddress) const; - - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -135,7 +124,7 @@ private: /// emitted for the start of the BB. More entries may be emitted to cover /// the location of calls or any instruction that may change control flow. void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress); + uint64_t FuncInputAddress, uint64_t FuncOutputAddress); /// Write the serialized address translation table for a function. template @@ -158,10 +147,6 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; - std::unordered_map> FuncHashes; - /// Map a function to its basic blocks count std::unordered_map NumBasicBlocksMap; @@ -174,6 +159,106 @@ private: /// Identifies the address of a control-flow changing instructions in a /// translation map entry const static uint32_t BRANCHENTRY = 0x1; + +public: + /// Map basic block input offset to a basic block index and hash pair. + class BBHashMapTy { + class EntryTy { + unsigned Index; + size_t Hash; + + public: + unsigned getBBIndex() const { return Index; } + size_t getBBHash() const { return Hash; } + EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint32_t BBInputOffset) const { + auto It = Map.find(BBInputOffset); + assert(It != Map.end()); + return It->second; + } + + public: + bool isInputBlock(uint32_t InputOffset) const { + return Map.count(InputOffset); + } + + unsigned getBBIndex(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBIndex(); + } + + size_t getBBHash(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBHash(); + } + + void addEntry(uint32_t BBInputOffset, unsigned BBIndex, size_t BBHash) { + Map.emplace(BBInputOffset, EntryTy(BBIndex, BBHash)); + } + + size_t getNumBasicBlocks() const { return Map.size(); } + }; + + /// Map function output address to its hash and basic blocks hash map. + class FuncHashesTy { + class EntryTy { + size_t Hash; + BBHashMapTy BBHashMap; + + public: + size_t getBFHash() const { return Hash; } + const BBHashMapTy &getBBHashMap() const { return BBHashMap; } + EntryTy(size_t Hash) : Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint64_t FuncOutputAddress) const { + auto It = Map.find(FuncOutputAddress); + assert(It != Map.end()); + return It->second; + } + + public: + size_t getBFHash(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBFHash(); + } + + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBBHashMap(); + } + + void addEntry(uint64_t FuncOutputAddress, size_t BFHash) { + Map.emplace(FuncOutputAddress, EntryTy(BFHash)); + } + + size_t getNumFunctions() const { return Map.size(); }; + + size_t getNumBasicBlocks() const { + size_t NumBasicBlocks{0}; + for (auto &I : Map) + NumBasicBlocks += I.second.getBBHashMap().getNumBasicBlocks(); + return NumBasicBlocks; + } + }; + + /// Returns BF hash by function output address (after BOLT). + size_t getBFHash(uint64_t FuncOutputAddress) const { + return FuncHashes.getBFHash(FuncOutputAddress); + } + + /// Returns BBHashMap by function output address (after BOLT). + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return FuncHashes.getBBHashMap(FuncOutputAddress); + } + + BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) { + return const_cast( + std::as_const(*this).getBBHashMap(FuncOutputAddress)); + } + +private: + FuncHashesTy FuncHashes; }; } // namespace bolt diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 31886f4c8025..57d2aff71977 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -22,12 +22,10 @@ const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress) { - uint64_t HotFuncAddress = ColdPartSource.count(FuncAddress) - ? ColdPartSource[FuncAddress] - : FuncAddress; + uint64_t FuncInputAddress, + uint64_t FuncOutputAddress) { const uint64_t BBOutputOffset = - BB.getOutputAddressRange().first - FuncAddress; + BB.getOutputAddressRange().first - FuncOutputAddress; const uint32_t BBInputOffset = BB.getInputOffset(); // Every output BB must track back to an input BB for profile collection @@ -42,11 +40,14 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << "BB " << BB.getName() << "\n"); LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(BBOutputOffset) << " Val: " << Twine::utohexstr(BBInputOffset) << "\n"); - LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", - getBBHash(HotFuncAddress, BBInputOffset))); - (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); + // NB: in `writeEntriesForBB` we use the input address because hashes are + // saved early in `saveMetadata` before output addresses are assigned. + const BBHashMapTy &BBHashMap = getBBHashMap(FuncInputAddress); + (void)BBHashMap; + LLVM_DEBUG( + dbgs() << formatv(" Hash: {0:x}\n", BBHashMap.getBBHash(BBInputOffset))); + LLVM_DEBUG( + dbgs() << formatv(" Index: {0}\n", BBHashMap.getBBIndex(BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -63,7 +64,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const auto InputAddress = BB.getFunction()->getAddress() + InputOffset; const auto OutputAddress = IOAddressMap.lookup(InputAddress); assert(OutputAddress && "Unknown instruction address"); - const auto OutputOffset = *OutputAddress - FuncAddress; + const auto OutputOffset = *OutputAddress - FuncOutputAddress; // Is this the first instruction in the BB? No need to duplicate the entry. if (OutputOffset == BBOutputOffset) @@ -99,7 +100,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { MapTy Map; for (const BinaryBasicBlock *const BB : Function.getLayout().getMainFragment()) - writeEntriesForBB(Map, *BB, Function.getOutputAddress()); + writeEntriesForBB(Map, *BB, InputAddress, OutputAddress); Maps.emplace(Function.getOutputAddress(), std::move(Map)); ReverseMap.emplace(OutputAddress, InputAddress); @@ -113,7 +114,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); Map.clear(); for (const BinaryBasicBlock *const BB : FF) - writeEntriesForBB(Map, *BB, FF.getAddress()); + writeEntriesForBB(Map, *BB, InputAddress, FF.getAddress()); Maps.emplace(FF.getAddress(), std::move(Map)); } @@ -125,11 +126,9 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { writeMaps(Maps, PrevAddress, OS); BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; - const uint64_t NumBBHashes = std::accumulate( - FuncHashes.begin(), FuncHashes.end(), 0ull, - [](size_t Acc, const auto &B) { return Acc + B.second.second.size(); }); - BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.size() << " function and " - << NumBBHashes << " basic block hashes\n"; + BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.getNumFunctions() + << " function and " << FuncHashes.getNumBasicBlocks() + << " basic block hashes\n"; } APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map, @@ -176,11 +175,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Only process cold fragments in cold mode, and vice versa. if (Cold != ColdPartSource.count(Address)) continue; - // NB: here we use the input address because hashes are saved early (in - // `saveMetadata`) before output addresses are assigned. + // NB: in `writeMaps` we use the input address because hashes are saved + // early in `saveMetadata` before output addresses are assigned. const uint64_t HotInputAddress = ReverseMap[Cold ? ColdPartSource[Address] : Address]; - std::pair &FuncHashPair = FuncHashes[HotInputAddress]; MapTy &Map = MapEntry.second; const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" @@ -194,10 +192,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, PrevIndex = HotIndex; } else { // Function hash - LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); - OS.write(reinterpret_cast(&FuncHashPair.first), 8); + size_t BFHash = getBFHash(HotInputAddress); + LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", BFHash)); + OS.write(reinterpret_cast(&BFHash), 8); // Number of basic blocks - size_t NumBasicBlocks = FuncHashPair.second.size(); + size_t NumBasicBlocks = getBBHashMap(HotInputAddress).getNumBasicBlocks(); LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); encodeULEB128(NumBasicBlocks, OS); } @@ -221,6 +220,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, }); } } + const BBHashMapTy &BBHashMap = getBBHashMap(HotInputAddress); size_t Index = 0; uint64_t InOffset = 0; size_t PrevBBIndex = 0; @@ -233,9 +233,9 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + const bool IsBlock = BBHashMap.isInputBlock(InOffset >> 1); + unsigned BBIndex = IsBlock ? BBHashMap.getBBIndex(InOffset >> 1) : 0; + size_t BBHash = IsBlock ? BBHashMap.getBBHash(InOffset >> 1) : 0; OS.write(reinterpret_cast(&BBHash), 8); // Basic block index in the input binary encodeULEB128(BBIndex - PrevBBIndex, OS); @@ -295,7 +295,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, HotFuncs.push_back(Address); // Function hash const size_t FuncHash = DE.getU64(&Offset, &Err); - FuncHashes[Address].first = FuncHash; + FuncHashes.addEntry(Address, FuncHash); LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); // Number of basic blocks const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); @@ -355,8 +355,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, BBIndexDelta = DE.getULEB128(&Offset, &Err); BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + getBBHashMap(HotAddress).addEntry(InputOffset >> 1, BBIndex, BBHash); } LLVM_DEBUG({ dbgs() << formatv( @@ -385,6 +384,8 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { OS << formatv(", hash: {0:x}", getBFHash(Address)); OS << "\n"; OS << "BB mappings:\n"; + const BBHashMapTy &BBHashMap = + getBBHashMap(HotAddress ? HotAddress : Address); for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit @@ -393,8 +394,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { if (IsBranch) OS << " (branch)"; else - OS << formatv(" hash: {0:x}", - getBBHash(HotAddress ? HotAddress : Address, Val)); + OS << formatv(" hash: {0:x}", BBHashMap.getBBHash(Val)); OS << "\n"; } OS << "\n"; @@ -515,27 +515,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple())) continue; // Prepare function and block hashes - FuncHashes[BF.getAddress()].first = BF.computeHash(); + FuncHashes.addEntry(BF.getAddress(), BF.computeHash()); BF.computeBlockHashes(); + BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress()); + // Set BF/BB metadata for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - -size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; -} - -size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { - return FuncHashes.at(OutputAddress).first; -} - } // namespace bolt } // namespace llvm -- GitLab From 90a7fc366ad05a7dd0465730c055af57c80d62d9 Mon Sep 17 00:00:00 2001 From: "Felix (Ting Wang)" Date: Sun, 24 Mar 2024 08:46:45 +0800 Subject: [PATCH 259/933] [PowerPC][NFC] Add base test case for small-local-dynamic-tls on AIX (#84711) --- ...aix-small-local-dynamic-tls-largeaccess.ll | 632 ++++++++++ .../aix-small-local-dynamic-tls-types.ll | 1066 +++++++++++++++++ 2 files changed, 1698 insertions(+) create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll new file mode 100644 index 000000000000..eb16bae67150 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll @@ -0,0 +1,632 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 + +; Test disassembly of object. +; RUN: llc -verify-machineinstrs -mcpu=pwr7 \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff -xcoff-traceback-table=false \ +; RUN: --code-model=large -filetype=obj -o %t.o < %s +; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck -D#NFA=2 --check-prefix=DIS %s + +@ElementIntTLSv1 = thread_local(localdynamic) global [8187 x i32] zeroinitializer, align 4 ; Within 32K +@ElementIntTLS2 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS3 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS4 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS5 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLSv2 = thread_local(localdynamic) global [9000 x i32] zeroinitializer, align 4 ; Beyond 32K + +@ElementLongTLS6 = external thread_local(localdynamic) global [60 x i64], align 8 +@ElementLongTLS2 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 ; Within 32K +@MyTLSGDVar = thread_local global [800 x i64] zeroinitializer, align 8 +@ElementLongTLS3 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS4 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS5 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS = thread_local(localdynamic) local_unnamed_addr global [7800 x i64] zeroinitializer, align 8 ; Beyond 32K + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16. +define signext i32 @test1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C5(r2) # target-flags(ppc-tlsld) @ElementIntTLSv1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv1) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [8187 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add6 = add i32 %add, %load2 + %add8 = add i32 %add6, %load3 + %add10 = add i32 %add8, %load4 + %add12 = add i32 %add10, %load5 + ret i32 %add12 +} + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16ds. +define i64 @test2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @ElementLongTLS6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 212 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 424(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C7(r2) # target-flags(ppc-tlsld) @ElementLongTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C8(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C10(r2) # target-flags(ppc-tlsld) @ElementLongTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @ElementLongTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 100 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 6800(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C12(r2) # target-flags(ppc-tlsld) @ElementLongTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 212 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C6@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 424(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C8@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C10@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 100 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C11@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 6800(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C12@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS6) + %arrayidx = getelementptr inbounds [60 x i64], ptr %tls1, i64 0, i64 53 + store i64 212, ptr %arrayidx, align 8 + %tls2 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS2) + %arrayidx1 = getelementptr inbounds [3000 x i64], ptr %tls2, i64 0, i64 150 + store i64 203, ptr %arrayidx1, align 8 + %tls3 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @MyTLSGDVar) + %arrayidx2 = getelementptr inbounds [800 x i64], ptr %tls3, i64 0, i64 55 + store i64 44, ptr %arrayidx2, align 8 + %tls4 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS3) + %arrayidx3 = getelementptr inbounds [3000 x i64], ptr %tls4, i64 0, i64 250 + store i64 6, ptr %arrayidx3, align 8 + %tls5 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS4) + %arrayidx4 = getelementptr inbounds [3000 x i64], ptr %tls5, i64 0, i64 850 + store i64 100, ptr %arrayidx4, align 8 + %tls6 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS5) + %arrayidx5 = getelementptr inbounds [3000 x i64], ptr %tls6, i64 0, i64 1050 + store i64 882, ptr %arrayidx5, align 8 + %load1 = load i64, ptr %arrayidx1, align 8 + %load2 = load i64, ptr %arrayidx3, align 8 + %load3 = load i64, ptr %arrayidx4, align 8 + %add = add i64 %load1, 882 + %add9 = add i64 %add, %load2 + %add11 = add i64 %add9, %load3 + ret i64 %add11 +} + +; Example of one access using the regular local-dynamic access from the TOC. +define signext i32 @test3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C13(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C13@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv2) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [9000 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add9 = add i32 %add, %load2 + %add11 = add i32 %add9, %load3 + %add13 = add i32 %add11, %load4 + %add15 = add i32 %add13, %load5 + ret i32 %add15 +} + +; DIS: file format aix5coff64-rs6000 +; DIS: Disassembly of section .text: +; DIS: 0000000000000000 (idx: [[#NFA+9]]) .test1: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 8(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000090 (idx: [[#NFA+11]]) .test2: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 212 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mr 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 48(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 424(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 203 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 56(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 1200(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 4, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 64(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 4, 72(4) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+3]]) .__tls_get_addr[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 44 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 440(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 80(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 2000(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 100 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 88(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 6800(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 882 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 96(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 8400(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 1191 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000140 (idx: [[#NFA+13]]) .test3: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 104(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: Disassembly of section .data: + +; DIS: 00000000000001d0 (idx: 17) test1[DS]: +; DIS-NEXT: 1d0: 00 00 00 00 +; DIS-NEXT: 00000000000001d0: R_POS (idx: [[#NFA+9]]) .test1 +; DIS-NEXT: 1d4: 00 00 00 00 +; DIS-NEXT: 1d8: 00 00 00 00 +; DIS-NEXT: 00000000000001d8: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1dc: 00 00 02 18 + +; DIS: 00000000000001e8 (idx: 19) test2[DS]: +; DIS-NEXT: 1e8: 00 00 00 00 +; DIS-NEXT: 00000000000001e8: R_POS (idx: [[#NFA+11]]) .test2 +; DIS-NEXT: 1ec: 00 00 00 90 +; DIS-NEXT: 1f0: 00 00 00 00 +; DIS-NEXT: 00000000000001f0: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1f4: 00 00 02 18 + +; DIS: 0000000000000200 (idx: 21) test3[DS]: +; DIS-NEXT: 200: 00 00 00 00 +; DIS-NEXT: 0000000000000200: R_POS (idx: [[#NFA+13]]) .test3 +; DIS-NEXT: 204: 00 00 01 40 +; DIS-NEXT: 208: 00 00 00 00 +; DIS-NEXT: 0000000000000208: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 20c: 00 00 02 18 + +; DIS: 0000000000000218 (idx: 25) _$TLSML[TC]: +; DIS-NEXT: 218: 00 00 00 00 +; DIS-NEXT: 0000000000000218: R_TLSML (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: 21c: 00 00 00 00 + +; DIS: 0000000000000220 (idx: 27) ElementIntTLSv1[TE]: +; DIS-NEXT: 220: 00 00 00 00 +; DIS-NEXT: 0000000000000220: R_TLS_LD (idx: [[#NFA+51]]) ElementIntTLSv1[TL] +; DIS-NEXT: 224: 00 00 00 00 + +; DIS: 0000000000000228 (idx: 29) ElementIntTLS3[TE]: +; DIS-NEXT: 228: 00 00 00 00 +; DIS-NEXT: 0000000000000228: R_TLS_LD (idx: [[#NFA+55]]) ElementIntTLS3[TL] +; DIS-NEXT: 22c: 00 00 be 6c + +; DIS: 0000000000000230 (idx: 31) ElementIntTLS4[TE]: +; DIS-NEXT: 230: 00 00 00 00 +; DIS-NEXT: 0000000000000230: R_TLS_LD (idx: [[#NFA+57]]) ElementIntTLS4[TL] +; DIS-NEXT: 234: 00 00 fc ec + +; DIS: 0000000000000238 (idx: 33) ElementIntTLS5[TE]: +; DIS-NEXT: 238: 00 00 00 00 +; DIS-NEXT: 0000000000000238: R_TLS_LD (idx: [[#NFA+59]]) ElementIntTLS5[TL] +; DIS-NEXT: 23c: 00 01 3b 6c + +; DIS: 0000000000000240 (idx: 35) ElementIntTLS2[TE]: +; DIS-NEXT: 240: 00 00 00 00 +; DIS-NEXT: 0000000000000240: R_TLS_LD (idx: [[#NFA+53]]) ElementIntTLS2[TL] +; DIS-NEXT: 244: 00 00 7f ec + +; DIS: 0000000000000248 (idx: 37) ElementLongTLS6[TE]: +; DIS-NEXT: 248: 00 00 00 00 +; DIS-NEXT: 0000000000000248: R_TLS_LD (idx: [[#NFA+5]]) ElementLongTLS6[UL] +; DIS-NEXT: 24c: 00 00 00 00 + +; DIS: 0000000000000250 (idx: 39) ElementLongTLS2[TE]: +; DIS-NEXT: 250: 00 00 00 00 +; DIS-NEXT: 0000000000000250: R_TLS_LD (idx: [[#NFA+63]]) ElementLongTLS2[TL] +; DIS-NEXT: 254: 00 02 06 90 + +; DIS: 0000000000000258 (idx: 41) .MyTLSGDVar[TE]: +; DIS-NEXT: 258: 00 00 00 00 +; DIS-NEXT: 0000000000000258: R_TLSM (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 25c: 00 00 00 00 + +; DIS: 0000000000000260 (idx: 43) MyTLSGDVar[TE]: +; DIS-NEXT: 260: 00 00 00 00 +; DIS-NEXT: 0000000000000260: R_TLS (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 264: 00 02 64 50 + +; DIS: 0000000000000268 (idx: 45) ElementLongTLS3[TE]: +; DIS-NEXT: 268: 00 00 00 00 +; DIS-NEXT: 0000000000000268: R_TLS_LD (idx: [[#NFA+67]]) ElementLongTLS3[TL] +; DIS-NEXT: 26c: 00 02 7d 50 + +; DIS: 0000000000000270 (idx: 47) ElementLongTLS4[TE]: +; DIS-NEXT: 270: 00 00 00 00 +; DIS-NEXT: 0000000000000270: R_TLS_LD (idx: [[#NFA+69]]) ElementLongTLS4[TL] +; DIS-NEXT: 274: 00 02 db 10 + +; DIS: 0000000000000278 (idx: 49) ElementLongTLS5[TE]: +; DIS-NEXT: 278: 00 00 00 00 +; DIS-NEXT: 0000000000000278: R_TLS_LD (idx: [[#NFA+71]]) ElementLongTLS5[TL] +; DIS-NEXT: 27c: 00 03 38 d0 + +; DIS: 0000000000000280 (idx: 51) ElementIntTLSv2[TE]: +; DIS-NEXT: 280: 00 00 00 00 +; DIS-NEXT: 0000000000000280: R_TLS_LD (idx: [[#NFA+61]]) ElementIntTLSv2[TL] +; DIS-NEXT: 284: 00 01 79 ec + +; DIS: Disassembly of section .tdata: +; DIS: 0000000000000000 (idx: [[#NFA+51]]) ElementIntTLSv1[TL]: +; DIS: 0000000000007fec (idx: [[#NFA+53]]) ElementIntTLS2[TL]: +; DIS: 000000000000be6c (idx: [[#NFA+55]]) ElementIntTLS3[TL]: +; DIS: 000000000000fcec (idx: [[#NFA+57]]) ElementIntTLS4[TL]: +; DIS: 0000000000013b6c (idx: [[#NFA+59]]) ElementIntTLS5[TL]: +; DIS: 00000000000179ec (idx: [[#NFA+61]]) ElementIntTLSv2[TL]: +; DIS: 0000000000020690 (idx: [[#NFA+63]]) ElementLongTLS2[TL]: +; DIS: 0000000000026450 (idx: [[#NFA+65]]) MyTLSGDVar[TL]: +; DIS: 0000000000027d50 (idx: [[#NFA+67]]) ElementLongTLS3[TL]: +; DIS: 000000000002db10 (idx: [[#NFA+69]]) ElementLongTLS4[TL]: +; DIS: 00000000000338d0 (idx: [[#NFA+71]]) ElementLongTLS5[TL]: +; DIS: 0000000000039690 (idx: [[#NFA+73]]) ElementLongTLS[TL]: diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll new file mode 100644 index 000000000000..d996d86a23d8 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll @@ -0,0 +1,1066 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0 + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 +@tlv_int_init = local_unnamed_addr global i32 87, align 4 + +@tlv_char = thread_local(localdynamic) global i8 1, align 1 +@tlv_short = thread_local(localdynamic) global i8 1, align 2 +@tlv_int = thread_local(localdynamic) global i32 1, align 4 +@internal_tlv_int = internal thread_local(localdynamic) global i32 1, align 4 +@tlv_long = thread_local(localdynamic) global i64 1, align 8 +@internal_tlv_long = internal thread_local(localdynamic) global i64 1, align 8 +@tlv_float = thread_local(localdynamic) global float 1.000000e+00, align 4 +@internal_tlv_double = internal thread_local(localdynamic) global double 1.000000e+00, align 8 + +%struct.anon = type { i32 } +@ThreadLocalStruct = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1 +@a = thread_local(localdynamic) global [87 x i32] zeroinitializer, align 4 + +define nonnull ptr @AddrTest1() local_unnamed_addr { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C0@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a) + %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 3 + ret ptr %arrayidx +} + +define signext i32 @testUnaligned() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C2@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C2@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStruct) + %x = getelementptr inbounds %struct.anon, ptr %tlv_addr, i32 0, i32 0 + %value = load i32, ptr %x, align 1 + ret i32 %value +} + +define void @testChar(i8 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C3@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C3@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @tlv_char) + store i8 %x, ptr %tlv_addr, align 1 + ret void +} + +define void @testShort(i16 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C4@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C4@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 2 ptr @llvm.threadlocal.address.p0(ptr align 2 @tlv_short) + store i16 %x, ptr %tlv_addr, align 2 + ret void +} + +define signext i32 @testInt1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C5@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_int) + %value = load i32, ptr %tlv_addr, align 4 + ret i32 %value +} + +define signext i32 @testInt2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C6@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C7@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C6@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @internal_tlv_int) + %tlv_val = load i32, ptr %tlv_addr, align 4 + %global_val = load i32, ptr @tlv_int_init, align 4 + %sum = add nsw i32 %global_val, %tlv_val + ret i32 %sum +} + +define signext i64 @testLong1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 4 + ret i64 %value +} + +define void @testLong2(i64 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %add = add nsw i64 %value, 9 + store i64 %add, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testLong3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %conv = trunc i64 %value to i32 + ret i32 %conv +} + +define void @testFloat1(float noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C11(r2) # %const.1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C12(r2) # %const.0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C11@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C12@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %inc = fadd float %value, 1.000000e+00 + %add = fadd float %inc, 8.000000e+00 + store float %add, ptr %tlv_addr, align 4 + ret void +} + +define i32 @testFloat2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %conv = fptosi float %value to i32 + ret i32 %conv +} + +define void @testDouble1(double noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + store double %x, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testDouble2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r1, 52 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 52(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 96(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r1, 68 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 68(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 80 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + %value = load double, ptr %tlv_addr, align 8 + %conv = fptosi double %value to i32 + ret i32 %conv +} -- GitLab From a118769ec8dc94b8332fe50c9240fecc8065a417 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 23 Mar 2024 20:56:45 -0400 Subject: [PATCH 260/933] [gn] port 3bc71c2abf --- llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn index efb324713cfe..d533e79e6374 100644 --- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn @@ -65,6 +65,7 @@ write_cmake_config("lit_common_configured") { "SANITIZER_USE_STATIC_CXX_ABI_PYBOOL=False", "SANITIZER_USE_STATIC_LLVM_UNWINDER_PYBOOL=False", "COMPILER_RT_HAS_AARCH64_SME_PYBOOL=False", + "COMPILER_RT_DARWIN_LINKER_VERSION=", "COMPILER_RT_HAS_LLD_PYBOOL=True", "COMPILER_RT_HAS_GWP_ASAN_PYBOOL=False", "HAVE_RPC_XDR_H=0", -- GitLab From 8e698a1d8e6984e41d2fdf159393d01951d87c21 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Sun, 24 Mar 2024 00:56:59 +0000 Subject: [PATCH 261/933] [gn build] Port b68e2eba0bc8 --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index 3c8b80526773..ac111f902513 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -287,6 +287,7 @@ if (current_toolchain == default_toolchain) { "__algorithm/shift_right.h", "__algorithm/shuffle.h", "__algorithm/sift_down.h", + "__algorithm/simd_utils.h", "__algorithm/sort.h", "__algorithm/sort_heap.h", "__algorithm/stable_partition.h", -- GitLab From d7d2f7ca62400ed4a3f8f89062d2aeec61bd29d4 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 19:11:49 -0700 Subject: [PATCH 262/933] [BOLT] Emit intra-function control flow in YAMLBAT Attach branch counters to YAML profile, covering intra-function control flow. Depends on: https://github.com/llvm/llvm-project/pull/86353 Test Plan: Updated bolt/test/X86/bolt-address-translation-yaml.test Reviewers: rafaelauler, dcci, ayermolo, maksfb Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76911 --- .../bolt/Profile/BoltAddressTranslation.h | 5 ++ bolt/lib/Profile/DataAggregator.cpp | 46 +++++++++++++++++++ .../X86/bolt-address-translation-yaml.test | 14 +++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index f8c35f8066f7..51fdadce8085 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -257,6 +257,11 @@ public: std::as_const(*this).getBBHashMap(FuncOutputAddress)); } + /// Returns the number of basic blocks in a function. + size_t getNumBasicBlocks(uint64_t OutputAddress) const { + return NumBasicBlocksMap.at(OutputAddress); + } + private: FuncHashesTy FuncHashes; }; diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 37c637a44a0e..b0f1cccc3256 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -2310,6 +2310,52 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC, BP.Functions.emplace_back( YAMLProfileWriter::convert(Function, /*UseDFS=*/false)); } + + for (const auto &KV : NamesToBranches) { + const StringRef FuncName = KV.first; + const FuncBranchData &Branches = KV.second; + yaml::bolt::BinaryFunctionProfile YamlBF; + BinaryData *BD = BC.getBinaryDataByName(FuncName); + assert(BD); + uint64_t FuncAddress = BD->getAddress(); + if (!BAT->isBATFunction(FuncAddress)) + continue; + // Filter out cold fragments + if (!BD->getSectionName().equals(BC.getMainCodeSectionName())) + continue; + BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress); + assert(BF); + YamlBF.Name = FuncName.str(); + YamlBF.Id = BF->getFunctionNumber(); + YamlBF.Hash = BAT->getBFHash(FuncAddress); + YamlBF.ExecCount = BF->getKnownExecutionCount(); + YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress); + const BoltAddressTranslation::BBHashMapTy &BlockMap = + BAT->getBBHashMap(FuncAddress); + + auto addSuccProfile = [&](yaml::bolt::BinaryBasicBlockProfile &YamlBB, + uint64_t SuccOffset, unsigned SuccDataIdx) { + const llvm::bolt::BranchInfo &BI = Branches.Data.at(SuccDataIdx); + yaml::bolt::SuccessorInfo SI; + SI.Index = BlockMap.getBBIndex(SuccOffset); + SI.Count = BI.Branches; + SI.Mispreds = BI.Mispreds; + YamlBB.Successors.emplace_back(SI); + }; + + for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) { + yaml::bolt::BinaryBasicBlockProfile YamlBB; + if (!BlockMap.isInputBlock(FromOffset)) + continue; + YamlBB.Index = BlockMap.getBBIndex(FromOffset); + YamlBB.Hash = BlockMap.getBBHash(FromOffset); + for (const auto &[SuccOffset, SuccDataIdx] : SuccKV) + addSuccProfile(YamlBB, SuccOffset, SuccDataIdx); + if (YamlBB.ExecCount || !YamlBB.Successors.empty()) + YamlBF.Blocks.emplace_back(YamlBB); + } + BP.Functions.emplace_back(YamlBF); + } } // Write the profile. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index ee54a90a9f2a..be3bcb2a0c6d 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -25,6 +25,7 @@ READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries YAML-BAT-CHECK: functions: +# Function not covered by BAT - has insns in basic block YAML-BAT-CHECK: - name: main YAML-BAT-CHECK-NEXT: fid: 2 YAML-BAT-CHECK-NEXT: hash: 0x9895746D48B2C876 @@ -35,6 +36,17 @@ YAML-BAT-CHECK-NEXT: - bid: 0 YAML-BAT-CHECK-NEXT: insns: 26 YAML-BAT-CHECK-NEXT: hash: 0xA900AE79CFD40000 YAML-BAT-CHECK-NEXT: succ: [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ] +# Function covered by BAT - doesn't have insns in basic block +YAML-BAT-CHECK: - name: usqrt +YAML-BAT-CHECK-NEXT: fid: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0x99E67ED32A203023 +YAML-BAT-CHECK-NEXT: exec: 21 +YAML-BAT-CHECK-NEXT: nblocks: 5 +YAML-BAT-CHECK-NEXT: blocks: +YAML-BAT-CHECK: - bid: 1 +YAML-BAT-CHECK-NEXT: insns: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0xD70DC695320E0010 +YAML-BAT-CHECK-NEXT: succ: {{.*}} { bid: 2, cnt: [[#]] } CHECK-BOLT-YAML: pre-processing profile using YAML profile reader -CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile +CHECK-BOLT-YAML-NEXT: 5 out of 16 functions in the binary (31.2%) have non-empty execution profile -- GitLab From 7c9b5228da94a44f5e3948814d896de537d162bb Mon Sep 17 00:00:00 2001 From: Owen Anderson Date: Sat, 23 Mar 2024 21:49:29 -0500 Subject: [PATCH 263/933] Only check assertions that were meant to apply to the normal case of non-splat vector SREM expansion when we aren't hitting the special case. (#86238) Fixes https://github.com/llvm/llvm-project/issues/84830 Introduced in https://github.com/llvm/llvm-project/pull/82706 --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++----- llvm/test/CodeGen/AArch64/srem-vec-crash.ll | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/srem-vec-crash.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index da29b1d5b312..8be03b66e155 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -6916,6 +6916,11 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // Q = floor((2 * A) / (2^K)) APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K)); + assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && + "We are expecting that A is always less than all-ones for SVT"); + assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && + "We are expecting that K is always less than all-ones for ShSVT"); + // If D was a power of two, apply the alternate constant derivation. if (D0.isOne()) { // A = 2^(W-1) @@ -6924,11 +6929,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, Q = APInt::getAllOnes(W - K).zext(W); } - assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && - "We are expecting that A is always less than all-ones for SVT"); - assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && - "We are expecting that K is always less than all-ones for ShSVT"); - // If the divisor is 1 the result can be constant-folded. Likewise, we // don't care about INT_MIN lanes, those can be set to undef if appropriate. if (D.isOne()) { diff --git a/llvm/test/CodeGen/AArch64/srem-vec-crash.ll b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll new file mode 100644 index 000000000000..0fce8de30d4d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s + +define i32 @pr84830(i1 %arg) { +; CHECK-LABEL: pr84830: +; CHECK: // %bb.0: // %bb +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +bb: + %new0 = srem i1 %arg, true + %last = zext i1 %new0 to i32 + %i = icmp ne i32 %last, 0 + %i1 = select i1 %i, i32 0, i32 1 + ret i32 %i1 +} -- GitLab From 74799f424063a2d751e0f9ea698db1f4efd0d8b2 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 23 Mar 2024 19:50:15 -0700 Subject: [PATCH 264/933] [memprof] Add call stack IDs to IndexedAllocationInfo (#85888) The indexed MemProf file has a huge amount of redundancy. In a large internal application, 82% of call stacks, stored in IndexedAllocationInfo::CallStack, are duplicates. We should work toward deduplicating call stacks by referring to them with unique IDs with actual call stacks stored in a separate data structure, much like we refer to memprof::Frame with memprof::FrameId. At the same time, we need to facilitate a graceful transition from the current version of the MemProf format to the next. We should be able to read (but not write) the current version of the MemProf file even after we move onto the next one. With those goals in mind, I propose to have an integer ID next to CallStack in IndexedAllocationInfo to refer to a call stack in a succinct manner. We'll gradually increase the areas of the compiler where IDs and call stacks have one-to-one correspondence and eventually remove the existing CallStack field. This patch adds call stack ID, named CSId, to IndexedAllocationInfo and teaches the raw profile reader to compute unique call stack IDs and store them in the new field. It does not introduce any user of the call stack IDs yet, except in verifyFunctionProfileData. --- llvm/include/llvm/ProfileData/MemProf.h | 23 ++++++++++++++++-- llvm/lib/ProfileData/MemProf.cpp | 25 ++++++++++++++++++++ llvm/lib/ProfileData/RawMemProfReader.cpp | 6 ++++- llvm/unittests/ProfileData/InstrProfTest.cpp | 3 ++- llvm/unittests/ProfileData/MemProfTest.cpp | 7 ++++-- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 37c19094bc2a..75ea0e49d453 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -1,6 +1,7 @@ #ifndef LLVM_PROFILEDATA_MEMPROF_H_ #define LLVM_PROFILEDATA_MEMPROF_H_ +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/GlobalValue.h" @@ -252,18 +253,26 @@ struct Frame { } }; +// A type representing the index into the table of call stacks. +using CallStackId = uint64_t; + // Holds allocation information in a space efficient format where frames are // represented using unique identifiers. struct IndexedAllocationInfo { // The dynamic calling context for the allocation in bottom-up (leaf-to-root) // order. Frame contents are stored out-of-line. + // TODO: Remove once we fully transition to CSId. llvm::SmallVector CallStack; + // Conceptually the same as above. We are going to keep both CallStack and + // CallStackId while we are transitioning from CallStack to CallStackId. + CallStackId CSId = 0; // The statistics obtained from the runtime for the allocation. PortableMemInfoBlock Info; IndexedAllocationInfo() = default; - IndexedAllocationInfo(ArrayRef CS, const MemInfoBlock &MB) - : CallStack(CS.begin(), CS.end()), Info(MB) {} + IndexedAllocationInfo(ArrayRef CS, CallStackId CSId, + const MemInfoBlock &MB) + : CallStack(CS.begin(), CS.end()), CSId(CSId), Info(MB) {} // Returns the size in bytes when this allocation info struct is serialized. size_t serializedSize() const { @@ -622,6 +631,16 @@ public: return Frame::deserialize(D); } }; + +// Compute a CallStackId for a given call stack. +CallStackId hashCallStack(ArrayRef CS); + +// Verify that each CallStackId is computed with hashCallStack. This function +// is intended to help transition from CallStack to CSId in +// IndexedAllocationInfo. +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData); } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 0461f0e9f840..bffa4ed19be8 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -3,8 +3,10 @@ #include "llvm/IR/Function.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/BLAKE3.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/HashBuilder.h" namespace llvm { namespace memprof { @@ -117,5 +119,28 @@ Expected readMemProfSchema(const unsigned char *&Buffer) { return Result; } +CallStackId hashCallStack(ArrayRef CS) { + llvm::HashBuilder, llvm::endianness::little> + HashBuilder; + for (FrameId F : CS) + HashBuilder.add(F); + llvm::BLAKE3Result<8> Hash = HashBuilder.final(); + CallStackId CSId; + std::memcpy(&CSId, Hash.data(), sizeof(Hash)); + return CSId; +} + +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData) { + for (const auto &[GUID, Record] : FunctionProfileData) { + (void)GUID; + for (const auto &AS : Record.AllocSites) { + assert(AS.CSId == hashCallStack(AS.CallStack)); + (void)AS; + } + } +} + } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp index 60c37c417aa0..5dc1ff897815 100644 --- a/llvm/lib/ProfileData/RawMemProfReader.cpp +++ b/llvm/lib/ProfileData/RawMemProfReader.cpp @@ -446,6 +446,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { Callstack.append(Frames.begin(), Frames.end()); } + CallStackId CSId = hashCallStack(Callstack); + // We attach the memprof record to each function bottom-up including the // first non-inline frame. for (size_t I = 0; /*Break out using the condition below*/; I++) { @@ -453,7 +455,7 @@ Error RawMemProfReader::mapRawProfileToRecords() { auto Result = FunctionProfileData.insert({F.Function, IndexedMemProfRecord()}); IndexedMemProfRecord &Record = Result.first->second; - Record.AllocSites.emplace_back(Callstack, Entry.second); + Record.AllocSites.emplace_back(Callstack, CSId, Entry.second); if (!F.IsInlineFrame) break; @@ -471,6 +473,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { } } + verifyFunctionProfileData(FunctionProfileData); + return Error::success(); } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index cd4552a039b3..c9323420bda7 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -366,7 +366,8 @@ IndexedMemProfRecord makeRecord( const MemInfoBlock &Block = MemInfoBlock()) { llvm::memprof::IndexedMemProfRecord MR; for (const auto &Frames : AllocFrames) - MR.AllocSites.emplace_back(Frames, Block); + MR.AllocSites.emplace_back(Frames, llvm::memprof::hashCallStack(Frames), + Block); for (const auto &Frames : CallSiteFrames) MR.CallSites.push_back(Frames); return MR; diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index f5e4a4aff2ed..1cca44e9b037 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -280,7 +280,8 @@ TEST(MemProf, RecordSerializationRoundTrip) { IndexedMemProfRecord Record; for (const auto &ACS : AllocCallStacks) { // Use the same info block for both allocation sites. - Record.AllocSites.emplace_back(ACS, Info); + Record.AllocSites.emplace_back(ACS, llvm::memprof::hashCallStack(ACS), + Info); } Record.CallSites.assign(CallSites); @@ -376,7 +377,9 @@ TEST(MemProf, BaseMemProfReader) { Block.AllocCount = 1U, Block.TotalAccessDensity = 4, Block.TotalLifetime = 200001; std::array CallStack{F1.hash(), F2.hash()}; - FakeRecord.AllocSites.emplace_back(/*CS=*/CallStack, /*MB=*/Block); + FakeRecord.AllocSites.emplace_back( + /*CS=*/CallStack, /*CSId=*/llvm::memprof::hashCallStack(CallStack), + /*MB=*/Block); ProfData.insert({F1.hash(), FakeRecord}); MemProfReader Reader(FrameIdMap, ProfData); -- GitLab From a45e58af1b381cf3c0374332386b8291ec5310f4 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Sun, 24 Mar 2024 12:48:19 +0900 Subject: [PATCH 265/933] [mlir][bufferization] Add `BufferViewFlowOpInterface` (#78718) This commit adds the `BufferViewFlowOpInterface` to the bufferization dialect. This interface can be implemented by ops that operate on buffers to indicate that a buffer op result and/or region entry block argument may be the same buffer as a buffer operand (or a view thereof). This interface is queried by the `BufferViewFlowAnalysis`. The new interface has two interface methods: * `populateDependencies`: Implementations use the provided callback to declare dependencies between operands and op results/region entry block arguments. E.g., for `%r = arith.select %c, %m1, %m2 : memref<5xf32>`, the interface implementation should declare two dependencies: %m1 -> %r and %m2 -> %r. * `mayBeTerminalBuffer`: An SSA value is a terminal buffer if the buffer view flow analysis stops at the specified value. E.g., because the value is a newly allocated buffer or because no further information is available about the origin of the buffer. Ops that implement the `RegionBranchOpInterface` or `BranchOpInterface` do not have to implement the `BufferViewFlowOpInterface`. The buffer dependencies can be inferred from those two interfaces. This commit makes the `BufferViewFlowAnalysis` more accurate. For unknown ops, it conservatively used to declare all combinations of operands and op results/region entry block arguments as dependencies (false positives). This is no longer the case. While the analysis is still a "maybe" analysis with false positives (e.g., when analyzing ops such as `arith.select` or `scf.if` where the taken branch is not known at compile time), results and region entry block arguments of unknown ops are now marked as terminal buffers. This commit addresses a TODO in `BufferViewFlowAnalysis.cpp`: ``` // TODO: We should have an op interface instead of a hard-coded list of // interfaces/ops. ``` It is no longer needed to hard-code ops. --- .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ .../IR/BufferViewFlowOpInterface.h | 27 +++++++ .../IR/BufferViewFlowOpInterface.td | 73 +++++++++++++++++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.h | 6 ++ .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ mlir/include/mlir/InitAllDialects.h | 4 + .../BufferViewFlowOpInterfaceImpl.cpp | 44 +++++++++++ .../Dialect/Arith/Transforms/CMakeLists.txt | 1 + .../IR/BufferViewFlowOpInterface.cpp | 18 +++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.cpp | 72 ++++++++++++++---- .../BufferViewFlowOpInterfaceImpl.cpp | 48 ++++++++++++ .../Dialect/MemRef/Transforms/CMakeLists.txt | 2 + .../llvm-project-overlay/mlir/BUILD.bazel | 40 ++++++++++ 15 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td create mode 100644 mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp create mode 100644 mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp create mode 100644 mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..a2b3a9bb655b --- /dev/null +++ b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace arith { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace arith +} // namespace mlir + +#endif // MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h new file mode 100644 index 000000000000..84e67fe72b62 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h @@ -0,0 +1,27 @@ +//===- BufferViewFlowOpInterface.h - Buffer View Flow Analysis --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ +#define MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ + +#include "mlir/IR/OpDefinition.h" +#include "mlir/Support/LLVM.h" + +namespace mlir { +class ValueRange; + +namespace bufferization { + +using RegisterDependenciesFn = std::function; + +} // namespace bufferization +} // namespace mlir + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc" + +#endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td new file mode 100644 index 000000000000..58885d742266 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td @@ -0,0 +1,73 @@ +//===-- BufferViewFlowOpInterface.td - Buffer View Flow ----*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef BUFFER_VIEW_FLOW_OP_INTERFACE +#define BUFFER_VIEW_FLOW_OP_INTERFACE + +include "mlir/IR/OpBase.td" + +def BufferViewFlowOpInterface : + OpInterface<"BufferViewFlowOpInterface"> { + let description = [{ + An op interface for the buffer view flow analysis. This interface describes + buffer dependencies between operands and op results/region entry block + arguments. + }]; + let cppNamespace = "::mlir::bufferization"; + let methods = [ + InterfaceMethod< + /*desc=*/[{ + Populate buffer dependencies between operands and op results/region + entry block arguments. + + Implementations should register dependencies between an operand ("X") + and an op result/region entry block argument ("Y") if Y may depend + on X. Y depends on X if Y and X are the same buffer or if Y is a + subview of X. + + Example: + ``` + %r = arith.select %c, %m1, %m2 : memref<5xf32> + ``` + In the above example, %0 may depend on %m1 or %m2 and a correct + interface implementation should call: + - "registerDependenciesFn(%m1, %r)". + - "registerDependenciesFn(%m2, %r)" + }], + /*retType=*/"void", + /*methodName=*/"populateDependencies", + /*args=*/(ins + "::mlir::bufferization::RegisterDependenciesFn" + :$registerDependenciesFn) + >, + InterfaceMethod< + /*desc=*/[{ + Return "true" if the given value may be a terminal buffer. A buffer + value is "terminal" if it cannot be traced back any further in the + buffer view flow analysis. + + Examples: A buffer could be terminal because: + - it is a newly allocated buffer (e.g., "memref.alloc"), + - or: because there is not enough compile-time information available + to make a definite decision (e.g., "memref.realloc" may reallocate + but we do not know for sure; another example are call ops where we + would have to analyze the body of the callee). + + Implementations can assume that the given SSA value is an OpResult of + this operation or a region entry block argument of this operation. + }], + /*retType=*/"bool", + /*methodName=*/"mayBeTerminalBuffer", + /*args=*/(ins "Value":$value), + /*methodBody=*/"", + /*defaultImplementation=*/"return false;" + >, + ]; +} + +#endif // BUFFER_VIEW_FLOW_OP_INTERFACE diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt index 31a553f9a32f..13a5bc370a4f 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt @@ -3,6 +3,7 @@ add_mlir_doc(BufferizationOps BufferizationOps Dialects/ -gen-dialect-doc) add_mlir_interface(AllocationOpInterface) add_mlir_interface(BufferDeallocationOpInterface) add_mlir_interface(BufferizableOpInterface) +add_mlir_interface(BufferViewFlowOpInterface) set(LLVM_TARGET_DEFINITIONS BufferizationEnums.td) mlir_tablegen(BufferizationEnums.h.inc -gen-enum-decls) diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 24825db69f90..9e43265c5dfe 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -63,6 +63,9 @@ public: /// results have to be changed. void rename(Value from, Value to); + /// Returns "true" if the given value may be a terminal. + bool mayBeTerminalBuffer(Value value) const; + private: /// This function constructs a mapping from values to its immediate /// dependencies. @@ -70,6 +73,9 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + + /// A set of all SSA values that may be terminal buffers. + DenseSet terminals; }; } // namespace mlir diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..714518a21e97 --- /dev/null +++ b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace memref { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace memref +} // namespace mlir + +#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 9bbf12d13254..c558dc53cc7f 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -21,6 +21,7 @@ #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.h" +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" #include "mlir/Dialect/ArmSME/IR/ArmSME.h" @@ -52,6 +53,7 @@ #include "mlir/Dialect/MemRef/IR/MemRefMemorySlot.h" #include "mlir/Dialect/MemRef/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.h" +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/RuntimeOpVerification.h" #include "mlir/Dialect/Mesh/IR/MeshDialect.h" #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" @@ -148,6 +150,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { affine::registerValueBoundsOpInterfaceExternalModels(registry); arith::registerBufferDeallocationOpInterfaceExternalModels(registry); arith::registerBufferizableOpInterfaceExternalModels(registry); + arith::registerBufferViewFlowOpInterfaceExternalModels(registry); arith::registerValueBoundsOpInterfaceExternalModels(registry); bufferization::func_ext::registerBufferizableOpInterfaceExternalModels( registry); @@ -157,6 +160,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { gpu::registerBufferDeallocationOpInterfaceExternalModels(registry); linalg::registerAllDialectInterfaceImplementations(registry); memref::registerAllocationOpInterfaceExternalModels(registry); + memref::registerBufferViewFlowOpInterfaceExternalModels(registry); memref::registerRuntimeVerifiableOpInterfaceExternalModels(registry); memref::registerValueBoundsOpInterfaceExternalModels(registry); memref::registerMemorySlotExternalModels(registry); diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..9df9df86b64f --- /dev/null +++ b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,44 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace arith { +namespace { + +struct SelectOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto selectOp = cast(op); + + // Either one of the true/false value may be selected at runtime. + registerDependenciesFn(selectOp.getTrueValue(), selectOp.getResult()); + registerDependenciesFn(selectOp.getFalseValue(), selectOp.getResult()); + } +}; + +} // namespace +} // namespace arith +} // namespace mlir + +void arith::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) { + SelectOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt index 02240601bcd3..12659eaba1fa 100644 --- a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_mlir_dialect_library(MLIRArithTransforms BufferDeallocationOpInterfaceImpl.cpp BufferizableOpInterfaceImpl.cpp Bufferize.cpp + BufferViewFlowOpInterfaceImpl.cpp EmulateUnsupportedFloats.cpp EmulateWideInt.cpp EmulateNarrowType.cpp diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp new file mode 100644 index 000000000000..ea726a4bfc3f --- /dev/null +++ b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp @@ -0,0 +1,18 @@ +//===- BufferViewFlowOpInterface.cpp - Buffer View Flow Analysis ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" + +namespace mlir { +namespace bufferization { + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc" + +} // namespace bufferization +} // namespace mlir diff --git a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt index 9895db9d93ce..63dcc1eb233e 100644 --- a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt @@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRBufferizationDialect BufferDeallocationOpInterface.cpp BufferizationOps.cpp BufferizationDialect.cpp + BufferViewFlowOpInterface.cpp UnstructuredControlFlow.cpp ADDITIONAL_HEADER_DIRS diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 88ef1b639fc5..9a36057425f3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -8,12 +8,16 @@ #include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Interfaces/CallInterfaces.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" +#include "mlir/Interfaces/FunctionInterfaces.h" #include "mlir/Interfaces/ViewLikeInterface.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" using namespace mlir; +using namespace mlir::bufferization; /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } @@ -65,18 +69,44 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) this->dependencies[value].insert(dep); }; + // Mark all buffer results and buffer region entry block arguments of the + // given op as terminals. + auto populateTerminalValues = [&](Operation *op) { + for (Value v : op->getResults()) + if (isa(v.getType())) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType())) + this->terminals.insert(v); + }; + op->walk([&](Operation *op) { - // TODO: We should have an op interface instead of a hard-coded list of - // interfaces/ops. + // Query BufferViewFlowOpInterface. If the op does not implement that + // interface, try to infer the dependencies from other interfaces that the + // op may implement. + if (auto bufferViewFlowOp = dyn_cast(op)) { + bufferViewFlowOp.populateDependencies(registerDependencies); + for (Value v : op->getResults()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + return WalkResult::advance(); + } // Add additional dependencies created by view changes to the alias list. if (auto viewInterface = dyn_cast(op)) { - dependencies[viewInterface.getViewSource()].insert( - viewInterface->getResult(0)); + registerDependencies(viewInterface.getViewSource(), + viewInterface->getResult(0)); return WalkResult::advance(); } @@ -131,16 +161,30 @@ void BufferViewFlowAnalysis::build(Operation *op) { return WalkResult::advance(); } - // Unknown op: Assume that all operands alias with all results. - for (Value operand : op->getOperands()) { - if (!isa(operand.getType())) - continue; - for (Value result : op->getResults()) { - if (!isa(result.getType())) - continue; - registerDependencies({operand}, {result}); - } + // Region terminators are handled together with RegionBranchOpInterface. + if (isa(op)) + return WalkResult::advance(); + + if (isa(op)) { + // This is an intra-function analysis. We have no information about other + // functions. Conservatively assume that each operand may alias with each + // result. Also mark the results are terminals because the function could + // return newly allocated buffers. + populateTerminalValues(op); + for (Value operand : op->getOperands()) + for (Value result : op->getResults()) + registerDependencies({operand}, {result}); + return WalkResult::advance(); } + + // We have no information about unknown ops. + populateTerminalValues(op); + return WalkResult::advance(); }); } + +bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { + assert(isa(value.getType()) && "expected memref"); + return terminals.contains(value); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..bbb269bd0016 --- /dev/null +++ b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,48 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace memref { +namespace { + +struct ReallocOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto reallocOp = cast(op); + // memref.realloc may return the source operand. + registerDependenciesFn(reallocOp.getSource(), reallocOp.getResult()); + } + + bool mayBeTerminalBuffer(Operation *op, Value value) const { + // The return value of memref.realloc is a terminal buffer because the op + // may return a newly allocated buffer. + return true; + } +}; + +} // namespace +} // namespace memref +} // namespace mlir + +void memref::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, memref::MemRefDialect *dialect) { + ReallocOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt index 08b7eab726eb..f150ac7ac2d6 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt @@ -1,5 +1,6 @@ add_mlir_dialect_library(MLIRMemRefTransforms AllocationOpInterfaceImpl.cpp + BufferViewFlowOpInterfaceImpl.cpp ComposeSubView.cpp ExpandOps.cpp ExpandRealloc.cpp @@ -27,6 +28,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms MLIRArithDialect MLIRArithTransforms MLIRBufferizationDialect + MLIRBufferizationTransforms MLIRDialectUtils MLIRFuncDialect MLIRGPUDialect diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 5b6e4678a05e..88b46bdb326c 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -10828,6 +10828,36 @@ gentbl_cc_library( ], ) +td_library( + name = "BufferViewFlowOpInterfaceTdFiles", + srcs = [ + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + ], + includes = ["include"], + deps = [ + ":OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "BufferViewFlowOpInterfaceIncGen", + tbl_outs = [ + ( + ["-gen-op-interface-decls"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc", + ), + ( + ["-gen-op-interface-defs"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + deps = [ + ":BufferViewFlowOpInterfaceTdFiles", + ], +) + td_library( name = "SubsetOpInterfaceTdFiles", srcs = [ @@ -12977,6 +13007,8 @@ cc_library( ":ArithTransforms", ":ArithUtils", ":BufferizationDialect", + ":BufferizationInterfaces", + ":BufferizationTransforms", ":ControlFlowDialect", ":DialectUtils", ":FuncDialect", @@ -13369,6 +13401,7 @@ td_library( includes = ["include"], deps = [ ":AllocationOpInterfaceTdFiles", + ":BufferViewFlowOpInterfaceTdFiles", ":BufferizableOpInterfaceTdFiles", ":CopyOpInterfaceTdFiles", ":DestinationStyleOpInterfaceTdFiles", @@ -13515,11 +13548,13 @@ cc_library( ], hdrs = [ "include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h", + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h", "include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h", ], includes = ["include"], deps = [ ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationEnumsIncGen", ":IR", @@ -13532,6 +13567,7 @@ cc_library( name = "BufferizationDialect", srcs = [ "lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp", + "lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizationDialect.cpp", "lib/Dialect/Bufferization/IR/BufferizationOps.cpp", @@ -13549,10 +13585,12 @@ cc_library( ":Analysis", ":ArithDialect", ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationBaseIncGen", ":BufferizationInterfaces", ":BufferizationOpsIncGen", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":CopyOpInterface", ":DestinationStyleOpInterface", @@ -13602,9 +13640,11 @@ cc_library( ":BufferizationDialect", ":BufferizationInterfaces", ":BufferizationPassIncGen", + ":CallOpInterfaces", ":ControlFlowDialect", ":ControlFlowInterfaces", ":FuncDialect", + ":FunctionInterfaces", ":IR", ":LoopLikeInterface", ":MemRefDialect", -- GitLab From 5d7fd6a04a6748936dece9d90481b2ba4ec97e53 Mon Sep 17 00:00:00 2001 From: yingopq <115543042+yingopq@users.noreply.github.com> Date: Sun, 24 Mar 2024 14:35:42 +0800 Subject: [PATCH 266/933] [Mips] Restore wrong deletion of instruction 'and' in unsigned min/max processing. (#85902) Fix #61881 --- llvm/lib/Target/Mips/MipsExpandPseudo.cpp | 9 ++++ llvm/test/CodeGen/Mips/atomic-min-max.ll | 56 +++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp index bded59439a73..c30129743a96 100644 --- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp +++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp @@ -500,6 +500,15 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( .addReg(Incr, RegState::Kill) .addImm(ShiftImm); } + } else { + // and OldVal, OldVal, Mask + // and Incr, Incr, Mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal) + .addReg(OldVal) + .addReg(Mask); + BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr) + .addReg(Incr) + .addReg(Mask); } } // unsigned: sltu Scratch4, oldVal, Incr diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll index bc3643f3947a..a96581bdb39a 100644 --- a/llvm/test/CodeGen/Mips/atomic-min-max.ll +++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll @@ -2146,6 +2146,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB6_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -2186,6 +2188,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB6_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -2225,6 +2229,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB6_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -2263,6 +2269,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB6_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -2300,6 +2308,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB6_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -2417,6 +2427,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB6_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -2456,6 +2468,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB6_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -2655,6 +2669,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB7_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -2696,6 +2712,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB7_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -2735,6 +2753,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB7_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -2773,6 +2793,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB7_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -2810,6 +2832,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB7_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -2927,6 +2951,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB7_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -2966,6 +2992,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB7_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 @@ -4244,6 +4272,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB10_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -4284,6 +4314,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB10_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -4323,6 +4355,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB10_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -4361,6 +4395,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB10_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -4398,6 +4434,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB10_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -4515,6 +4553,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB10_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -4554,6 +4594,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB10_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -4753,6 +4795,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB11_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -4793,6 +4837,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB11_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -4832,6 +4878,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB11_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -4870,6 +4918,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB11_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -4907,6 +4957,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB11_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -5024,6 +5076,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB11_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -5063,6 +5117,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB11_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 -- GitLab From 4acd84e7ccce6a2865f60cd2adc37a335d4f35ce Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 24 Mar 2024 01:22:48 -0700 Subject: [PATCH 267/933] Revert "[compiler-rt] Also consider SIGPROF as a synchronous signal" (#86416) Reverting #85188 with follow up patches. This reverts commit 362d26366d0175f01ffb6085eb747a6e40f01147. This reverts commit c9bdeabdf4b46fbf1f6a9fcbf9cd61d460b18c08. This reverts commit 6bc6e1ace9fa8453e164fa04b5d9acd5a77e089a. This reverts commit 01fa550ff654d6724e6da54c877032baeddff14b. This reverts commit ddcbab37ac0e5743a8d39be3dd48d967f4c85504. --- compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 4 +--- compiler-rt/test/tsan/signal_errno.cpp | 6 +++--- compiler-rt/test/tsan/signal_reset.cpp | 8 ++++---- compiler-rt/test/tsan/signal_sync.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread2.cpp | 4 ++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 810ce69663d0..8ffc703b05ea 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -126,7 +126,6 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -const int SIGPROF = 27; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; @@ -2180,8 +2179,7 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { return; } // Don't mess with synchronous signals. - const bool sync = is_sync_signal(sctx, sig, info) || - (sig == SIGPROF && thr->is_inited && !thr->is_dead); + const bool sync = is_sync_signal(sctx, sig, info); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, diff --git a/compiler-rt/test/tsan/signal_errno.cpp b/compiler-rt/test/tsan/signal_errno.cpp index 99d4b6d84ca4..7e1fd4b0c5a5 100644 --- a/compiler-rt/test/tsan/signal_errno.cpp +++ b/compiler-rt/test/tsan/signal_errno.cpp @@ -18,7 +18,7 @@ static void MyHandler(int, siginfo_t *s, void *c) { static void* sendsignal(void *p) { barrier_wait(&barrier); - pthread_kill(mainth, SIGALRM); + pthread_kill(mainth, SIGPROF); return 0; } @@ -37,7 +37,7 @@ int main() { mainth = pthread_self(); struct sigaction act = {}; act.sa_sigaction = &MyHandler; - sigaction(SIGALRM, &act, 0); + sigaction(SIGPROF, &act, 0); pthread_t th; pthread_create(&th, 0, sendsignal, 0); loop(); @@ -46,7 +46,7 @@ int main() { } // CHECK: WARNING: ThreadSanitizer: signal handler spoils errno -// CHECK: Signal 14 handler invoked at: +// CHECK: Signal 27 handler invoked at: // CHECK: #0 MyHandler(int, {{(__)?}}siginfo{{(_t)?}}*, void*) {{.*}}signal_errno.cpp // CHECK: main // CHECK: SUMMARY: ThreadSanitizer: signal handler spoils errno{{.*}}MyHandler diff --git a/compiler-rt/test/tsan/signal_reset.cpp b/compiler-rt/test/tsan/signal_reset.cpp index d76b7e5f3b5f..82758d882382 100644 --- a/compiler-rt/test/tsan/signal_reset.cpp +++ b/compiler-rt/test/tsan/signal_reset.cpp @@ -28,12 +28,12 @@ static void* reset(void *p) { struct sigaction act = {}; for (int i = 0; i < 1000000; i++) { act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -44,7 +44,7 @@ static void* reset(void *p) { int main() { struct sigaction act = {}; act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -53,7 +53,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_sync.cpp b/compiler-rt/test/tsan/signal_sync.cpp index 878b3f3b88b9..b529a1859f52 100644 --- a/compiler-rt/test/tsan/signal_sync.cpp +++ b/compiler-rt/test/tsan/signal_sync.cpp @@ -30,7 +30,7 @@ int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -39,7 +39,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread.cpp b/compiler-rt/test/tsan/signal_thread.cpp index 7bba8159bf38..aa91d1ddeb10 100644 --- a/compiler-rt/test/tsan/signal_thread.cpp +++ b/compiler-rt/test/tsan/signal_thread.cpp @@ -24,7 +24,7 @@ static void* thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -33,7 +33,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread2.cpp b/compiler-rt/test/tsan/signal_thread2.cpp index 5236628e13b6..9bde4f70b39d 100644 --- a/compiler-rt/test/tsan/signal_thread2.cpp +++ b/compiler-rt/test/tsan/signal_thread2.cpp @@ -40,7 +40,7 @@ static void *thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -49,7 +49,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } -- GitLab From 4d69855e9d380ecb7c1f7a64c7b37258fe36f525 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Sun, 24 Mar 2024 06:48:04 -0400 Subject: [PATCH 268/933] [flang] Silence MSVC warning about shifts (NFC) (#83737) Yes, 64-bit shifts are intended. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index eb8f5135ff12..ea1ef1f08aba 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -3883,7 +3883,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, int pos = 3 + highSignificandSize; mlir::Value index = builder.create( loc, builder.create(loc, intVal, signShift), - createIntegerConstant(1 << pos)); + createIntegerConstant(1ULL << pos)); // [e] exponent != 0 mlir::Value exponent = @@ -3895,7 +3895,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::ne, exponent, zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [m] exponent == 1..1 (max exponent) index = builder.create( @@ -3904,7 +3904,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::eq, exponent, exponentMask), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [l] low-order significand != 0 index = builder.create( @@ -3916,7 +3916,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, builder.create(loc, intVal, lowSignificandMask), zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [h] high-order significand (1 or 2 bits) index = builder.create( -- GitLab From 6c6fe4b2aea8631001b11abee62146d4aca01cee Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 24 Mar 2024 11:33:51 +0000 Subject: [PATCH 269/933] [X86] known-never-zero.ll - add 32-bit test coverage Enabled vector coverage as well: i686+SSE2 and x64_64+AVX Should improve test quality for #85722 --- llvm/test/CodeGen/X86/known-never-zero.ll | 1708 ++++++++++++++------- 1 file changed, 1161 insertions(+), 547 deletions(-) diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll index cc9862769f2b..423516bc3271 100644 --- a/llvm/test/CodeGen/X86/known-never-zero.ll +++ b/llvm/test/CodeGen/X86/known-never-zero.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X86 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64 ;; Use cttz to test if we properly prove never-zero. There is a very ;; simple transform from cttz -> cttz_zero_undef if its operand is @@ -18,41 +19,70 @@ declare i32 @llvm.fshl.i32(i32, i32, i32) declare i32 @llvm.fshr.i32(i32, i32, i32) define i32 @or_known_nonzero(i32 %x) { -; CHECK-LABEL: or_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %z = or i32 %x, 1 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @or_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: or_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl %esi, %edi -; CHECK-NEXT: je .LBB1_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB1_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB1_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB1_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl %esi, %edi +; X64-NEXT: je .LBB1_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB1_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = or i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @select_known_nonzero(i1 %c, i32 %x) { -; CHECK-LABEL: select_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: movl $122, %eax -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: movl $122, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: testb $1, %dil +; X64-NEXT: movl $122, %eax +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 122 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -60,20 +90,36 @@ define i32 @select_known_nonzero(i1 %c, i32 %x) { } define i32 @select_maybe_zero(i1 %c, i32 %x) { -; CHECK-LABEL: select_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB3_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB3_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl $1, %ecx +; X86-NEXT: xorl %eax, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB3_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB3_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: testb $1, %dil +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB3_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB3_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 0 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -81,28 +127,45 @@ define i32 @select_maybe_zero(i1 %c, i32 %x) { } define i32 @shl_known_nonzero_1s_bit_set(i32 %x) { -; CHECK-LABEL: shl_known_nonzero_1s_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $123, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_1s_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $123, %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_1s_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $123, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = shl i32 123, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -110,14 +173,23 @@ define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -125,67 +197,116 @@ define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @shl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: shl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB7_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB7_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB7_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB7_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB7_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB7_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = shl nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_known_nonzero(i32 %x) { -; CHECK-LABEL: uaddsat_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: incl %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovnel %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: incl %eax +; X86-NEXT: movl $-1, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: incl %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovnel %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 1) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: uaddsat_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovael %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB9_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB9_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-1, %eax +; X86-NEXT: cmovael %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB9_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB9_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: addl %esi, %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovael %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB9_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB9_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umax_known_nonzero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cmpl %eax, %edi -; CHECK-NEXT: cmoval %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: cmpl %edx, %eax +; X86-NEXT: cmoval %eax, %edx +; X86-NEXT: rep bsfl %edx, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cmpl %eax, %edi +; X64-NEXT: cmoval %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %yy = shl nuw i32 4, %y %z = call i32 @llvm.umax.i32(i32 %x, i32 %yy) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -193,35 +314,62 @@ define i32 @umax_known_nonzero(i32 %x, i32 %y) { } define i32 @umax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl %esi, %edi -; CHECK-NEXT: cmoval %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB11_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB11_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl %eax, %ecx +; X86-NEXT: cmoval %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB11_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB11_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl %esi, %edi +; X64-NEXT: cmoval %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB11_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB11_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umax.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: umin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovbl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovbl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovbl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.umin.i32(i32 %x, i32 %y) @@ -230,36 +378,63 @@ define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @umin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovbl %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB13_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB13_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovbl %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB13_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB13_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovbl %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB13_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB13_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovll %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovll %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovll %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smin.i32(i32 %x, i32 %y) @@ -268,36 +443,63 @@ define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovll %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB15_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB15_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovll %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB15_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB15_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovll %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB15_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB15_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.smin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovgl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovgl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovgl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smax.i32(i32 %x, i32 %y) @@ -306,35 +508,61 @@ define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $55, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovgel %edi, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cmpl $55, %eax +; X86-NEXT: movl $54, %ecx +; X86-NEXT: cmovgel %eax, %ecx +; X86-NEXT: bsfl %ecx, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $55, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovgel %edi, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %z = call i32 @llvm.smax.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB18_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB18_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB18_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB18_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB18_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB18_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shr = lshr i32 %x, %y %sub = sub i32 32, %y @@ -345,19 +573,33 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB19_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB19_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB19_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB19_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB19_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB19_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shr = lshr i32 %x, %y %sub = sub i32 32, %y %shl = shl i32 %x, %sub @@ -367,14 +609,23 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) { } define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -382,39 +633,68 @@ define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB21_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB21_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB21_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB21_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB21_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB21_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB22_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB22_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB22_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB22_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB22_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB22_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shl = shl i32 %x, %y %sub = sub i32 32, %y @@ -425,19 +705,33 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB23_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB23_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB23_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB23_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB23_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB23_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shl = shl i32 %x, %y %sub = sub i32 32, %y %shr = lshr i32 %x, %sub @@ -447,14 +741,23 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) { } define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -462,47 +765,78 @@ define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB25_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB25_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB25_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB25_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB25_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB25_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: sra_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = ashr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: sra_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -510,47 +844,78 @@ define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @sra_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sra_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB28_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB28_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB28_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB28_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB28_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB28_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: srl_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = lshr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: srl_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -558,33 +923,56 @@ define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @srl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: srl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB31_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB31_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB31_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB31_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB31_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB31_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: udiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -592,33 +980,56 @@ define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @udiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: udiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB33_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB33_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB33_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB33_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB33_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB33_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: sdiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -626,31 +1037,53 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @sdiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sdiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB35_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB35_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB35_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB35_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB35_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB35_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @add_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nuw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -658,17 +1091,30 @@ define i32 @add_known_nonzero(i32 %xx, i32 %y) { } define i32 @add_maybe_zero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: je .LBB37_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB37_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB37_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB37_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: je .LBB37_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB37_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nsw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -676,15 +1122,24 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) { } define i32 @sub_known_nonzero_neg_case(i32 %xx) { -; CHECK-LABEL: sub_known_nonzero_neg_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: negl %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_neg_case: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: negl %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_neg_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: negl %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i32 256, %xx %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -692,14 +1147,24 @@ define i32 @sub_known_nonzero_neg_case(i32 %xx) { } define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { -; CHECK-LABEL: sub_known_nonzero_ne_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: andl $-65, %edi -; CHECK-NEXT: subl %eax, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_ne_case: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: orl $64, %ecx +; X86-NEXT: andl $-65, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_ne_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: andl $-65, %edi +; X64-NEXT: subl %eax, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %y = and i32 %xx, -65 %z = sub i32 %y, %x @@ -708,18 +1173,32 @@ define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { } define i32 @sub_maybe_zero(i32 %x) { -; CHECK-LABEL: sub_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: subl %edi, %eax -; CHECK-NEXT: je .LBB40_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB40_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: je .LBB40_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB40_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: subl %edi, %eax +; X64-NEXT: je .LBB40_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB40_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 64 %z = sub i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -727,34 +1206,60 @@ define i32 @sub_maybe_zero(i32 %x) { } define i32 @sub_maybe_zero2(i32 %x) { -; CHECK-LABEL: sub_maybe_zero2: -; CHECK: # %bb.0: -; CHECK-NEXT: negl %edi -; CHECK-NEXT: je .LBB41_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB41_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero2: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: negl %eax +; X86-NEXT: je .LBB41_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB41_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero2: +; X64: # %bb.0: +; X64-NEXT: negl %edi +; X64-NEXT: je .LBB41_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB41_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB42_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB42_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB42_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB42_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB42_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB42_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -762,18 +1267,32 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB43_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB43_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB43_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB43_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB43_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB43_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -781,36 +1300,63 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @mul_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: mul_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: imull %esi, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB44_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB44_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB44_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB44_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: imull %esi, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB44_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB44_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = mul nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_known_nonzero(<2 x i16> %xx) { -; CHECK-LABEL: bitcast_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; CHECK-NEXT: pslld $23, %xmm0 -; CHECK-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: cvttps2dq %xmm0, %xmm0 -; CHECK-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; CHECK-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; X86-NEXT: pslld $23, %xmm0 +; X86-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: cvttps2dq %xmm0, %xmm0 +; X86-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] +; X86-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: bsfl %eax, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; X64-NEXT: vpslld $23, %xmm0, %xmm0 +; X64-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vcvttps2dq %xmm0, %xmm0 +; X64-NEXT: vpackusdw %xmm0, %xmm0, %xmm0 +; X64-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %x = shl nuw nsw <2 x i16> , %xx %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -818,49 +1364,83 @@ define i32 @bitcast_known_nonzero(<2 x i16> %xx) { } define i32 @bitcast_maybe_zero(<2 x i16> %x) { -; CHECK-LABEL: bitcast_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB46_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB46_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB46_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB46_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB46_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB46_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_from_float(float %x) { -; CHECK-LABEL: bitcast_from_float: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB47_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB47_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_from_float: +; X86: # %bb.0: +; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB47_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB47_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_from_float: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB47_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB47_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast float %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @zext_known_nonzero(i16 %xx) { -; CHECK-LABEL: zext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: movzwl %ax, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -868,32 +1448,54 @@ define i32 @zext_known_nonzero(i16 %xx) { } define i32 @zext_maybe_zero(i16 %x) { -; CHECK-LABEL: zext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB49_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movzwl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB49_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testw %ax, %ax +; X86-NEXT: je .LBB49_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB49_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB49_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movzwl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB49_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sext_known_nonzero(i16 %xx) { -; CHECK-LABEL: sext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cwtl -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: cwtl +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cwtl +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -901,17 +1503,29 @@ define i32 @sext_known_nonzero(i16 %xx) { } define i32 @sext_maybe_zero(i16 %x) { -; CHECK-LABEL: sext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB51_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movswl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB51_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB51_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB51_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB51_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movswl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB51_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r -- GitLab From e8d5223ce4b2214e052b5b52b2e6453ffea0fe33 Mon Sep 17 00:00:00 2001 From: David Green Date: Sun, 24 Mar 2024 12:32:47 +0000 Subject: [PATCH 270/933] [AArch64] Additional GISel test coverage. NFC --- llvm/test/CodeGen/AArch64/setcc_knownbits.ll | 95 +++++++++++++------- 1 file changed, 65 insertions(+), 30 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll index bb9546af8bb7..6a337bf02e09 100644 --- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll +++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i1 @load_bv_v4i8(i1 zeroext %a) { ; CHECK-LABEL: load_bv_v4i8: @@ -11,18 +12,31 @@ define i1 @load_bv_v4i8(i1 zeroext %a) { } define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) { -; CHECK-LABEL: logger: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldr w8, [x2] -; CHECK-NEXT: cmp w8, w0 -; CHECK-NEXT: b.ls .LBB1_2 -; CHECK-NEXT: // %bb.1: -; CHECK-NEXT: mov w0, wzr -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB1_2: // %land.rhs -; CHECK-NEXT: ldr x8, [x1] -; CHECK-NEXT: ldrb w0, [x8] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: logger: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ldr w8, [x2] +; CHECK-SD-NEXT: cmp w8, w0 +; CHECK-SD-NEXT: b.ls .LBB1_2 +; CHECK-SD-NEXT: // %bb.1: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB1_2: // %land.rhs +; CHECK-SD-NEXT: ldr x8, [x1] +; CHECK-SD-NEXT: ldrb w0, [x8] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: logger: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr w8, [x2] +; CHECK-GI-NEXT: cmp w8, w0 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.hi .LBB1_2 +; CHECK-GI-NEXT: // %bb.1: // %land.rhs +; CHECK-GI-NEXT: ldr x8, [x1] +; CHECK-GI-NEXT: ldrb w8, [x8] +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: .LBB1_2: // %land.end +; CHECK-GI-NEXT: ret entry: %0 = load i32, ptr %pll, align 4 %cmp.not = icmp ugt i32 %0, %logLevel @@ -44,12 +58,20 @@ land.end: ; preds = %land.rhs, %entry declare i64 @llvm.ctlz.i64(i64 %in, i1) define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { -; CHECK-LABEL: lshr_ctlz_undef_cmpeq_one_i64: -; CHECK: // %bb.0: -; CHECK-NEXT: clz x8, x0 -; CHECK-NEXT: lsr x0, x8, #6 -; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: clz x8, x0 +; CHECK-SD-NEXT: lsr x0, x8, #6 +; CHECK-SD-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: clz x8, x0 +; CHECK-GI-NEXT: lsr x8, x8, #6 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1) %lshr = lshr i64 %ctlz, 6 %icmp = icmp eq i64 %lshr, 1 @@ -57,17 +79,30 @@ define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { } define i32 @PR17487(i1 %tobool) { -; CHECK-LABEL: PR17487: -; CHECK: // %bb.0: -; CHECK-NEXT: dup v0.2s, w0 -; CHECK-NEXT: mov w8, #1 // =0x1 -; CHECK-NEXT: dup v1.2d, x8 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-NEXT: bic v0.16b, v1.16b, v0.16b -; CHECK-NEXT: mov x8, v0.d[1] -; CHECK-NEXT: cmp x8, #1 -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: PR17487: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: dup v0.2s, w0 +; CHECK-SD-NEXT: mov w8, #1 // =0x1 +; CHECK-SD-NEXT: dup v1.2d, x8 +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-SD-NEXT: mov x8, v0.d[1] +; CHECK-SD-NEXT: cmp x8, #1 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: PR17487: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-GI-NEXT: mov v0.d[1], x0 +; CHECK-GI-NEXT: adrp x8, .LCPI3_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI3_0] +; CHECK-GI-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-GI-NEXT: mov d0, v0.d[1] +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret %tmp = insertelement <2 x i1> undef, i1 %tobool, i32 1 %tmp1 = zext <2 x i1> %tmp to <2 x i64> %tmp2 = xor <2 x i64> %tmp1, -- GitLab From b3fe27f2be0585d5d2ad46f96956ccfd76ca003e Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Sun, 24 Mar 2024 16:14:56 +0100 Subject: [PATCH 271/933] [InstCombine] Copy flags of extractelement for extelt -> icmp combine (#86366) Fixes #86164 --- llvm/include/llvm/IR/InstrTypes.h | 11 +++++++++++ llvm/lib/IR/Instructions.cpp | 10 ++++++++++ .../Transforms/InstCombine/InstCombineVectorOps.cpp | 4 +++- llvm/test/Transforms/InstCombine/scalarization.ll | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index e8c2cba8418d..d2b33fe9c651 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1058,6 +1058,17 @@ public: static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); + /// Construct a compare instruction, given the opcode, the predicate, + /// the two operands and the instruction to copy the flags from. Optionally + /// (if InstBefore is specified) insert the instruction into a BasicBlock + /// right before the specified instruction. The specified Instruction is + /// allowed to be a dereferenced end iterator. Create a CmpInst + static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name = "", + Instruction *InsertBefore = nullptr); + /// Get the opcode casted to the right type OtherOps getOpcode() const { return static_cast(Instruction::getOpcode()); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 494d50f89e37..8ca211e6e2e7 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -4623,6 +4623,16 @@ CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, S1, S2, Name); } +CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name, + Instruction *InsertBefore) { + CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore); + Inst->copyIRFlags(FlagsSource); + return Inst; +} + void CmpInst::swapOperands() { if (ICmpInst *IC = dyn_cast(this)) IC->swapOperands(); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c7f4fb17648c..99f1f8eb34bb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -487,7 +487,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index) Value *E0 = Builder.CreateExtractElement(X, Index); Value *E1 = Builder.CreateExtractElement(Y, Index); - return CmpInst::Create(cast(SrcVec)->getOpcode(), Pred, E0, E1); + CmpInst *SrcCmpInst = cast(SrcVec); + return CmpInst::CreateWithCopiedFlags(SrcCmpInst->getOpcode(), Pred, E0, E1, + SrcCmpInst); } if (auto *I = dyn_cast(SrcVec)) { diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll index fe6dc526bd50..7e645ef7e883 100644 --- a/llvm/test/Transforms/InstCombine/scalarization.ll +++ b/llvm/test/Transforms/InstCombine/scalarization.ll @@ -341,6 +341,17 @@ define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) { ret i1 %ext } +define i1 @extractelt_vector_fcmp_copy_flags(<4 x float> %x) { +; CHECK-LABEL: @extractelt_vector_fcmp_copy_flags( +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = fcmp nsz arcp oeq float [[TMP1]], 0.000000e+00 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %cmp = fcmp nsz arcp oeq <4 x float> %x, zeroinitializer + %r = extractelement <4 x i1> %cmp, i32 2 + ret i1 %r +} + define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) { ; ; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use( -- GitLab From 9632e1515c93453efc39752b1c9f32aedd358fbc Mon Sep 17 00:00:00 2001 From: houndlord <45179481+houndlord@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:33:16 +0100 Subject: [PATCH 272/933] Match fixed width ISD::AVGFLOORS + ISD::AVGCEILS patterns (#86222) --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++++++++++----- llvm/test/CodeGen/AArch64/hadd-combine.ll | 24 ++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index e27a8bb8fdac..05b4ce3aaa2c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2529,20 +2529,28 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); } -// Attempt to form avgceilu(A, B) from (A | B) - ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGCEILU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) { - SDValue A, B; if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) { + if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B); + } + } return SDValue(); } @@ -2837,20 +2845,29 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { return SDValue(); } -// Attempt to form avgflooru(A, B) from (A & B) + ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGFLOORU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) { - SDValue A, B; if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) { + if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B); + } + } + return SDValue(); } @@ -2869,8 +2886,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGFLOORU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGFLOORU(N, DAG)) + // Try to match AVGFLOOR fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG)) return V; // fold (a+b) -> (a|b) iff a and b share no bits. @@ -3868,8 +3885,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGCEILU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGCEILU(N, DAG)) + // Try to match AVGCEIL fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG)) return V; if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N))) diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll index e12502980790..491bf40ea4aa 100644 --- a/llvm/test/CodeGen/AArch64/hadd-combine.ll +++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll @@ -341,6 +341,18 @@ define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: srhadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %or = or <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = sub <8 x i16> %or, %srl + ret <8 x i16> %res +} + define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) { ; CHECK-LABEL: rhaddu_base: ; CHECK: // %bb.0: @@ -879,6 +891,18 @@ define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: shadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %and = and <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = add <8 x i16> %and, %srl + ret <8 x i16> %res +} + declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>) declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>) declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>) -- GitLab From 48048051323d5dd74057dc5f32df8c3c323afcd5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Sun, 24 Mar 2024 11:43:15 -0600 Subject: [PATCH 273/933] [lld/ELF][X86] Respect outSecOff when checking if GOTPCREL can be relaxed (#86334) The existing implementation didn't handle when the input text section was some offset from the output section. This resulted in an assert in relaxGot() with an lld built with asserts for some large binaries, or even worse, a silently broken binary with an lld without asserts. --- lld/ELF/Arch/X86_64.cpp | 7 ++++--- lld/test/ELF/x86-64-gotpc-relax-too-far.s | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index de459013595f..a85bf3aa0c09 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -328,9 +328,10 @@ bool X86_64::relaxOnce(int pass) const { if (rel.expr != R_RELAX_GOT_PC) continue; - uint64_t v = sec->getRelocTargetVA( - sec->file, rel.type, rel.addend, - sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr); + uint64_t v = sec->getRelocTargetVA(sec->file, rel.type, rel.addend, + sec->getOutputSection()->addr + + sec->outSecOff + rel.offset, + *rel.sym, rel.expr); if (isInt<32>(v)) continue; if (rel.sym->auxIdx == 0) { diff --git a/lld/test/ELF/x86-64-gotpc-relax-too-far.s b/lld/test/ELF/x86-64-gotpc-relax-too-far.s index 74aa6d8f65a0..ba41faab67de 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-too-far.s +++ b/lld/test/ELF/x86-64-gotpc-relax-too-far.s @@ -5,7 +5,10 @@ # RUN: llvm-objdump --no-print-imm-hex -d %t/bin | FileCheck --check-prefix=DISASM %s # RUN: llvm-readelf -S %t/bin | FileCheck --check-prefixes=GOT %s # RUN: ld.lld -T %t/lds2 %t/a.o -o %t/bin2 -# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=UNNECESSARY-GOT %s +# RUN: llvm-objdump --no-print-imm-hex -d %t/bin2 | FileCheck --check-prefix=DISASM %s +# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=GOT %s +# RUN: ld.lld -T %t/lds3 %t/a.o -o %t/bin3 +# RUN: llvm-readelf -S %t/bin3 | FileCheck --check-prefixes=UNNECESSARY-GOT %s # DISASM: <_foo>: # DISASM-NEXT: movl 2097146(%rip), %eax @@ -47,6 +50,13 @@ SECTIONS { data 0x80200000 : { *(data) } } #--- lds2 +SECTIONS { + .text.foo 0x100000 : { *(.text.foo) } + .text 0x1ff000 : { . = . + 0x1000 ; *(.text) } + .got 0x300000 : { *(.got) } + data 0x80200000 : { *(data) } +} +#--- lds3 SECTIONS { .text.foo 0x100000 : { *(.text.foo) } .text 0x200000 : { *(.text) } -- GitLab From d4a4585165f5c6ca8c42920b70e1b47696ff1172 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 24 Mar 2024 19:52:19 +0100 Subject: [PATCH 274/933] [clang-tidy] Ignore expresions in unevaluated context in bugprone-inc-dec-in-conditions (#85849) Skip checking for references to variable in unevaluated context, like decltype, static_assert and so on. Closes #85838 --- .../clang-tidy/bugprone/IncDecInConditionsCheck.cpp | 8 +++++++- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/bugprone/inc-dec-in-conditions.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index 16f43128d55e..9b3b01eb0268 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -31,6 +31,10 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), cxxOperatorCallExpr(isComparisonOperator()))); + auto IsInUnevaluatedContext = + expr(anyOf(hasAncestor(expr(matchers::hasUnevaluatedContext())), + hasAncestor(typeLoc()))); + Finder->addMatcher( expr( OperatorMatcher, unless(isExpansionInSystemHeader()), @@ -42,12 +46,14 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { cxxOperatorCallExpr( isPrePostOperator(), hasUnaryOperand(expr().bind("operand")))), + unless(IsInUnevaluatedContext), hasAncestor( expr(equalsBoundNode("parent"), hasDescendant( expr(unless(equalsBoundNode("operand")), matchers::isStatementIdenticalToBoundNode( - "operand")) + "operand"), + unless(IsInUnevaluatedContext)) .bind("second"))))) .bind("operator"))), this); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index a604e9276668..2392ccaf6575 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -139,6 +139,10 @@ Changes in existing checks ` check by detecting side effect from calling a method with non-const reference parameters. +- Improved :doc:`bugprone-inc-dec-in-conditions + ` check to ignore code + within unevaluated contexts, such as ``decltype``. + - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion ` check by eliminating false positives resulting from direct usage of bitwise operators diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp index 82af039973c3..91de013138f0 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp @@ -68,3 +68,13 @@ bool doubleCheck(Container x) { // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: decrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] // CHECK-MESSAGES: :[[@LINE-2]]:31: warning: incrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] } + +namespace PR85838 { + void test() + { + auto foo = 0; + auto bar = 0; + if (++foo < static_cast(bar)) {} + if (static_cast(bar) < foo) {} + } +} -- GitLab From 909ea28ac60760a0e9b39369b2f54a4e8f1daec9 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 14:29:23 -0500 Subject: [PATCH 275/933] [Libomptarget] Specificall add LLVM include dirs in plugins --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 0420d0e6f1f8..b84f3d7b137c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -60,6 +60,7 @@ target_link_options(PluginCommon PUBLIC ${offload_link_flags}) target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_LLVM_INCLUDE_DIRS} ${LIBOMPTARGET_INCLUDE_DIR} ) -- GitLab From 488a18738f5c275093bcc5459da69e9b4c9de074 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sun, 24 Mar 2024 10:41:21 -0500 Subject: [PATCH 276/933] [JITLink][ELF] Improve ELF section start/end symbol handling. This commit adds section start and stop symbol handling to ELF/aarch64, and fixes the section symbol prefixes (using `__start_` and `__stop_`, rather than `__start` and `__end`). It also adds a testcase for handling of these symbols. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 2 +- .../DefineExternalSectionStartAndEndSymbols.h | 19 ++++++++ .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 7 +++ .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 18 -------- .../ELF_section_start_and_stop_symbols.s | 43 +++++++++++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 30a9383dc0bc..4ce0c8d9fd17 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -567,7 +567,7 @@ public: orc::ExecutorAddrDiff getOffset() const { return Offset; } void setOffset(orc::ExecutorAddrDiff NewOffset) { - assert(NewOffset < getBlock().getSize() && "Offset out of range"); + assert(NewOffset <= getBlock().getSize() && "Offset out of range"); Offset = NewOffset; } diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 159880e4b152..aef441d093a2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -108,6 +108,25 @@ createDefineExternalSectionStartAndEndSymbolsPass( std::forward(F)); } +/// ELF section start/end symbol detection. +inline SectionRangeSymbolDesc +identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { + constexpr StringRef StartSymbolPrefix = "__start_"; + constexpr StringRef EndSymbolPrefix = "__stop_"; + + auto SymName = Sym.getName(); + if (SymName.starts_with(StartSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) + return {*Sec, true}; + } else if (SymName.starts_with(EndSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) + return {*Sec, false}; + } + return {}; +} + } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index f17b2c626ac2..4f64d83162b2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -20,6 +20,8 @@ #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" +#include "DefineExternalSectionStartAndEndSymbols.h" + #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -611,6 +613,11 @@ void link_ELF_aarch64(std::unique_ptr G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); + // Resolve any external section start / end symbols. + Config.PostAllocationPasses.push_back( + createDefineExternalSectionStartAndEndSymbolsPass( + identifyELFSectionStartAndEndSymbols)); + // Add an in-place GOT/TLS/Stubs build pass. Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index a1fe9c5fcd73..52dd83d9040f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -343,24 +343,6 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } -static SectionRangeSymbolDesc -identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { - constexpr StringRef StartSymbolPrefix = "__start"; - constexpr StringRef EndSymbolPrefix = "__end"; - - auto SymName = Sym.getName(); - if (SymName.starts_with(StartSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) - return {*Sec, true}; - } else if (SymName.starts_with(EndSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) - return {*Sec, false}; - } - return {}; -} - void link_ELF_x86_64(std::unique_ptr G, std::unique_ptr Ctx) { PassConfiguration Config; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s new file mode 100644 index 000000000000..f8e7ba96f006 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s @@ -0,0 +1,43 @@ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -position-independent \ +# RUN: -filetype=obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + + .text + .file "elf_section_start_stop.c" + .globl main + .p2align 2 + .type main,@function +main: + adrp x8, z + adrp x9, y + ldr w8, [x8, :lo12:z] + ldr w9, [x9, :lo12:y] + sub w0, w8, w9 + ret +.Lfunc_end0: + .size main, .Lfunc_end0-main + + .type x,@object + .section custom_section,"aw",@progbits + .globl x + .p2align 2 +x: + .word 42 + .size x, 4 + +# jitlink-check: *{8}z = (*{8}y) + 4 + + .type y,@object + .data + .globl y + .p2align 3, 0x0 +y: + .xword __start_custom_section + .size y, 8 + + .type z,@object + .globl z + .p2align 3, 0x0 +z: + .xword __stop_custom_section + .size z, 8 -- GitLab From 9f0321ccf118b37e5cb93cabd2acbf600c36b6ee Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 15:23:50 -0500 Subject: [PATCH 277/933] [Libomptarget] Make plugins depend explicitly on `intrinsics_gen` Summary: It's possible for the OpenMP offloading plugins to be build before tablegen is run despite the fact that we rely on it. Simply make it depend on it currently. --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index b84f3d7b137c..a7350e662a7c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(PluginCommon OBJECT src/RPC.cpp src/Utils/ELF.cpp ) +add_dependencies(PluginCommon intrinsics_gen) # Only enable JIT for those targets that LLVM can support. string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported) -- GitLab From 81e2693c1202d6c4e48dbf2d985153b03cfccb79 Mon Sep 17 00:00:00 2001 From: Hui Date: Sun, 24 Mar 2024 20:52:47 +0000 Subject: [PATCH 278/933] [libc++][test] Fix race condition in condition_variable_any tests (#84788) Some tests in `condition_variable_any` use two `shared_lock` to guard, which does not work. The fix is to make the writer to use `unique_lock` --- .../thread.condition.condvarany/wait_for_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_until_token_pred.pass.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp index 4ea60557d9f8..7a39d1253a33 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(2ms); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp index e96a3e8bd1bc..f322d8cfdc68 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp @@ -63,7 +63,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp index d649db025d75..e7388b9ce0e1 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); -- GitLab From f0a8738401137c32f3e1b9799244ed643feb7bf0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 24 Mar 2024 20:59:53 +0000 Subject: [PATCH 279/933] [VPlan] Generate CalculateTripCountMinusVF for Part 0 only. (NFCI). The value produced by CalculateTripCountMinusVF VPInstructions is independent of the part. Only compute it for part 0 and use that for other parts. --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 + .../AArch64/scalable-strict-fadd.ll | 63 +++---------------- .../AArch64/sve-tail-folding-unroll.ll | 42 ++----------- .../AArch64/uniform-args-call-variants.ll | 13 +--- 4 files changed, 21 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d75e322a74cf..f91997a37993 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -345,6 +345,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name); } case VPInstruction::CalculateTripCountMinusVF: { + if (Part != 0) + return State.get(this, 0, /*IsScalar*/ true); + Value *ScalarTC = State.get(getOperand(0), {0, 0}); Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index fc67fb5aded6..ad6e8534f318 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -403,21 +403,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -492,9 +477,9 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP78:%.*]] = mul i64 [[TMP77]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP79:%.*]] = add i64 [[INDEX]], [[TMP78]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP80:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP81:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP82:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1715,21 +1700,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -1826,9 +1796,9 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -2129,21 +2099,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -2240,9 +2195,9 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index 1a6e83a61ce7..2acc1ddbffea 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -25,21 +25,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -107,9 +92,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 12 ; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP72:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP73:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP74:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -167,21 +152,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -275,9 +245,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP92:%.*]] = mul i64 [[TMP91]], 12 ; CHECK-NEXT: [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP94:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP95:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT14]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP96:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT15]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 4957bbeda671..d8f14f30295b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -40,9 +40,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -71,7 +68,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -129,9 +126,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -160,7 +154,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -207,7 +201,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) -; INTERLEAVE-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1 ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -237,7 +230,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 ; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP0]] ; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: ; INTERLEAVE-NEXT: ret void -- GitLab From 18a49f03aa2b6bfeb073648b9eb75277a2386fc4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2024 14:07:09 -0700 Subject: [PATCH 280/933] [ELF] Merge relaIplt into relaDyn `relaIplt` was added so that IRELATIVE relocations are placed at the end of .rela.dyn (since https://reviews.llvm.org/D65651) or .rela.plt (--pack-dyn-relocs=android[+relr]). Unfortunately, handling `relaIplt` requires special cases all over the code base. We can extend partitionRels/computeRels to partition both RELATIVE and IRELATIVE relocations, rendering `relaIplt` unneeded. The change allows IRELATIVE relocations in the DT_ANDROID_REL[A] table (untested?!), which may be processed before other types of relocations. This seems acceptable for Bionic's DEFINE_IFUNC_FOR use cases. In addition, this change simplies changing .rel[a].dyn to a compact relocation format (CREL). SHF_INFO_LINK is removed from .rel[a].dyn with IRELATIVE relocations. (See https://reviews.llvm.org/D89828). --- lld/ELF/Relocations.cpp | 10 ++--- lld/ELF/SyntheticSections.cpp | 19 +++----- lld/ELF/SyntheticSections.h | 1 - lld/ELF/Writer.cpp | 50 ++++++--------------- lld/test/ELF/aarch64-gnu-ifunc.s | 3 +- lld/test/ELF/arm-gnu-ifunc.s | 3 +- lld/test/ELF/gnu-ifunc-i386.s | 3 +- lld/test/ELF/systemz-ifunc-nonpreemptible.s | 2 +- 8 files changed, 28 insertions(+), 63 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 92a1b9baaca3..33c50133bec4 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1620,12 +1620,8 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // relatively straightforward. We create a PLT entry in Iplt, which is // usually at the end of .plt, which makes an indirect call using a // matching GOT entry in igotPlt, which is usually at the end of .got.plt. - // The GOT entry is relocated using an IRELATIVE relocation in relaIplt, - // which is usually at the end of .rela.plt. Unlike most relocations in - // .rela.plt, which may be evaluated lazily without -z now, dynamic - // loaders evaluate IRELATIVE relocs eagerly, which means that for - // IRELATIVE relocs only, GOT-generating relocations can point directly to - // .got.plt without requiring a separate GOT entry. + // The GOT entry is relocated using an IRELATIVE relocation in relaDyn, + // which is usually at the end of .rela.dyn. // // - Despite the fact that an ifunc does not have a fixed value, compilers // that are not passed -fPIC will assume that they do, and will emit @@ -1665,7 +1661,7 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // section/value fixed. auto *directSym = makeDefined(cast(sym)); directSym->allocateAux(); - addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, + addPltEntry(*in.iplt, *in.igotPlt, *mainPart->relaDyn, target->iRelativeRel, *directSym); sym.allocateAux(); symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 10eda17f0cb3..f924756ddddf 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1267,15 +1267,12 @@ DynamicSection::DynamicSection() // The output section .rela.dyn may include these synthetic sections: // // - part.relaDyn -// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn // - in.relaPlt: this is included if a linker script places .rela.plt inside // .rela.dyn // // DT_RELASZ is the total size of the included sections. static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { size_t size = relaDyn.getSize(); - if (in.relaIplt->getParent() == relaDyn.getParent()) - size += in.relaIplt->getSize(); if (in.relaPlt->getParent() == relaDyn.getParent()) size += in.relaPlt->getSize(); return size; @@ -1372,9 +1369,7 @@ DynamicSection::computeContents() { if (!config->shared && !config->relocatable && !config->zRodynamic) addInt(DT_DEBUG, 0); - if (part.relaDyn->isNeeded() || - (in.relaIplt->isNeeded() && - part.relaDyn->getParent() == in.relaIplt->getParent())) { + if (part.relaDyn->isNeeded()) { addInSec(part.relaDyn->dynamicTag, *part.relaDyn); entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(*part.relaDyn)); @@ -1657,10 +1652,6 @@ void RelocationBaseSection::finalizeContents() { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.gotPlt->getParent()->sectionIndex; } - if (in.relaIplt.get() == this && in.igotPlt->getParent()) { - getParent()->flags |= ELF::SHF_INFO_LINK; - getParent()->info = in.igotPlt->getParent()->sectionIndex; - } } void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) { @@ -1674,6 +1665,11 @@ void RelocationBaseSection::computeRels() { SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); parallelForEach(relocs, [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); }); + + auto irelative = std::partition( + relocs.begin() + numRelativeRelocs, relocs.end(), + [t = target->iRelativeRel](auto &r) { return r.type != t; }); + // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset // is to make results easier to read. @@ -1682,7 +1678,7 @@ void RelocationBaseSection::computeRels() { parallelSort(relocs.begin(), nonRelative, [&](auto &a, auto &b) { return a.r_offset < b.r_offset; }); // Non-relative relocations are few, so don't bother with parallelSort. - llvm::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) { + llvm::sort(nonRelative, irelative, [&](auto &a, auto &b) { return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset); }); } @@ -3843,7 +3839,6 @@ void InStruct::reset() { ppc32Got2.reset(); ibtPlt.reset(); relaPlt.reset(); - relaIplt.reset(); shStrTab.reset(); strTab.reset(); symTab.reset(); diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index b41e69410054..fa21b80a5a5e 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -1358,7 +1358,6 @@ struct InStruct { std::unique_ptr ppc32Got2; std::unique_ptr ibtPlt; std::unique_ptr relaPlt; - std::unique_ptr relaIplt; std::unique_ptr shStrTab; std::unique_ptr strTab; std::unique_ptr symTab; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 4eca7b22e90b..40d617b7fdf3 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -449,8 +449,8 @@ template void elf::createSyntheticSections() { add(*part.dynamic); add(*part.dynStrTab); - add(*part.relaDyn); } + add(*part.relaDyn); if (config->relrPackDynRelocs) { part.relrDyn = std::make_unique>(threadCount); @@ -550,17 +550,6 @@ template void elf::createSyntheticSections() { /*threadCount=*/1); add(*in.relaPlt); - // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative - // relocations are processed last by the dynamic loader. We cannot place the - // iplt section in .rel.dyn when Android relocation packing is enabled because - // that would cause a section type mismatch. However, because the Android - // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired - // behaviour by placing the iplt section in .rel.plt. - in.relaIplt = std::make_unique>( - config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, - /*sort=*/false, /*threadCount=*/1); - add(*in.relaIplt); - if ((config->emachine == EM_386 || config->emachine == EM_X86_64) && (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) { in.ibtPlt = std::make_unique(); @@ -1071,20 +1060,18 @@ void PhdrEntry::add(OutputSection *sec) { sec->ptLoad = this; } -// The beginning and the ending of .rel[a].plt section are marked -// with __rel[a]_iplt_{start,end} symbols if it is a statically linked -// executable. The runtime needs these symbols in order to resolve -// all IRELATIVE relocs on startup. For dynamic executables, we don't -// need these symbols, since IRELATIVE relocs are resolved through GOT -// and PLT. For details, see http://www.airs.com/blog/archives/403. +// A statically linked position-dependent executable should only contain +// IRELATIVE relocations and no other dynamic relocations. Encapsulation symbols +// __rel[a]_iplt_{start,end} will be defined for .rel[a].dyn, to be +// processed by the libc runtime. Other executables or DSOs use dynamic tags +// instead. template void Writer::addRelIpltSymbols() { if (config->isPic) return; - // By default, __rela_iplt_{start,end} belong to a dummy section 0 - // because .rela.plt might be empty and thus removed from output. - // We'll override Out::elfHeader with In.relaIplt later when we are - // sure that .rela.plt exists in output. + // __rela_iplt_{start,end} are initially defined relative to dummy section 0. + // We'll override Out::elfHeader with relaDyn later when we are sure that + // .rela.dyn will be present in the output. ElfSym::relaIpltStart = addOptionalRegular( config->isRela ? "__rela_iplt_start" : "__rel_iplt_start", Out::elfHeader, 0, STV_HIDDEN); @@ -1110,11 +1097,11 @@ template void Writer::setReservedSymbolSections() { ElfSym::globalOffsetTable->section = sec; } - // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. - if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { - ElfSym::relaIpltStart->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->value = in.relaIplt->getSize(); + // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn. + if (ElfSym::relaIpltStart && mainPart->relaDyn->isNeeded()) { + ElfSym::relaIpltStart->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->value = mainPart->relaDyn->getSize(); } PhdrEntry *last = nullptr; @@ -1470,14 +1457,6 @@ static void sortSection(OutputSection &osec, if (name == ".init" || name == ".fini") return; - // IRelative relocations that usually live in the .rel[a].dyn section should - // be processed last by the dynamic loader. To achieve that we add synthetic - // sections in the required order from the beginning so that the in.relaIplt - // section is placed last in an output section. Here we just do not apply - // sorting for an output section which holds the in.relaIplt section. - if (in.relaIplt->getParent() == &osec) - return; - // Sort input sections by priority using the list provided by // --symbol-ordering-file or --shuffle-sections=. This is a least significant // digit radix sort. The sections may be sorted stably again by a more @@ -2196,7 +2175,6 @@ template void Writer::finalizeSections() { finalizeSynthetic(in.mipsGot.get()); finalizeSynthetic(in.igotPlt.get()); finalizeSynthetic(in.gotPlt.get()); - finalizeSynthetic(in.relaIplt.get()); finalizeSynthetic(in.relaPlt.get()); finalizeSynthetic(in.plt.get()); finalizeSynthetic(in.iplt.get()); diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s index dee24779d913..d76b54eabf8a 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc.s +++ b/lld/test/ELF/aarch64-gnu-ifunc.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_RELA // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0x158 // CHECK-NEXT: Size: 48 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 8 // CHECK-NEXT: EntrySize: 24 // CHECK-NEXT: } diff --git a/lld/test/ELF/arm-gnu-ifunc.s b/lld/test/ELF/arm-gnu-ifunc.s index 562478256fd8..d49ca18e991e 100644 --- a/lld/test/ELF/arm-gnu-ifunc.s +++ b/lld/test/ELF/arm-gnu-ifunc.s @@ -30,13 +30,12 @@ _start: // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: 0x100F4 // CHECK-NEXT: Offset: 0xF4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK: Name: .iplt // CHECK-NEXT: Type: SHT_PROGBITS // CHECK-NEXT: Flags [ diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s index b502fd6e9ae2..43b19b27ea4e 100644 --- a/lld/test/ELF/gnu-ifunc-i386.s +++ b/lld/test/ELF/gnu-ifunc-i386.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0xD4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 4 // CHECK-NEXT: EntrySize: 8 // CHECK-NEXT: } diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s index 5056db302ca1..892bbde8d9c7 100644 --- a/lld/test/ELF/systemz-ifunc-nonpreemptible.s +++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s @@ -10,7 +10,7 @@ # CHECK: Section Headers: # CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al # CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 -# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 AI 0 4 8 +# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 A 0 0 8 # CHECK-NEXT: [ 2] .text PROGBITS 0000000001001188 000188 00001c 00 AX 0 0 4 # CHECK-NEXT: [ 3] .iplt PROGBITS 00000000010011b0 0001b0 000040 00 AX 0 0 16 # CHECK-NEXT: [ 4] .got.plt PROGBITS 00000000010021f0 0001f0 000010 00 WA 0 0 8 -- GitLab From 67f2267ae006f35edfdf36567646403e61527d1b Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Sun, 24 Mar 2024 14:24:44 -0700 Subject: [PATCH 281/933] [M68k][NFC] Suppress warning on an unused variable when assertion is disabled NFC. --- llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp index 158393f02a24..786aa7bcb64e 100644 --- a/llvm/lib/Target/M68k/M68kISelLowering.cpp +++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp @@ -939,6 +939,7 @@ SDValue M68kTargetLowering::LowerFormalArguments( for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; assert(VA.getValNo() != LastVal && "Same value in different locations"); + (void)LastVal; LastVal = VA.getValNo(); -- GitLab From cceedc939a43c7c732a5888364251775bffc2dba Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sun, 24 Mar 2024 15:22:40 -0700 Subject: [PATCH 282/933] [clang-format] Fix a crash with AlignArrayOfStructures option (#86420) Fixes #86109. --- clang/lib/Format/WhitespaceManager.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index fef85abf79a3..710bf8d8a8ec 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -1491,7 +1491,7 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start, : Cell); // Go to the next non-comment and ensure there is a break in front const auto *NextNonComment = C.Tok->getNextNonComment(); - while (NextNonComment->is(tok::comma)) + while (NextNonComment && NextNonComment->is(tok::comma)) NextNonComment = NextNonComment->getNextNonComment(); auto j = i; while (j < End && Changes[j].Tok != NextNonComment) diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index cf8d6ab691d9..03005384a6f6 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -21100,7 +21100,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresRightAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", " @@ -21353,7 +21360,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresLeftAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", {\"x\", " -- GitLab From 230b1895c493c511c11541af3b5bc819887c82a8 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Sun, 24 Mar 2024 18:56:47 -0700 Subject: [PATCH 283/933] [mlir][spirv] Add folding for [S|U|LessThan[Equal] (#85435) Add missing constant propogation folder for [S|U]LessThan[Equal]. Implement additional folding when the operands are equal for all ops. Allows for constant folding in the IndexToSPIRV pass. Part of work #70704 --- .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td | 8 + .../SPIRV/IR/SPIRVCanonicalization.cpp | 82 ++++++++ .../SPIRV/Transforms/canonicalize.mlir | 176 ++++++++++++++++++ 3 files changed, 266 insertions(+) diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td index 3ee239d6e1e3..14d639bc26f2 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td @@ -716,6 +716,8 @@ def SPIRV_SLessThanOp : SPIRV_LogicalBinaryOp<"SLessThan", ``` }]; + + let hasFolder = 1; } // ----- @@ -745,6 +747,8 @@ def SPIRV_SLessThanEqualOp : SPIRV_LogicalBinaryOp<"SLessThanEqual", %5 = spirv.SLessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -886,6 +890,8 @@ def SPIRV_ULessThanOp : SPIRV_LogicalBinaryOp<"ULessThan", %5 = spirv.ULessThan %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -949,6 +955,8 @@ def SPIRV_ULessThanEqualOp : SPIRV_LogicalBinaryOp<"ULessThanEqual", %5 = spirv.ULessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } #endif // MLIR_DIALECT_SPIRV_IR_LOGICAL_OPS diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index ff4bace9a4d8..3f1a7826f8b6 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -880,6 +880,88 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) { }); } +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::SLessThanOp::fold(spirv::SLessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.slt(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::SLessThanEqualOp::fold(spirv::SLessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.sle(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ULessThanOp::fold(spirv::ULessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ult(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::ULessThanEqualOp::fold(spirv::ULessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ule(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + //===----------------------------------------------------------------------===// // spirv.ShiftLeftLogical //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index de21d114e9fc..ed0bd070c43c 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> { // ----- +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @slt_same +func.func @slt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.SLessThan %arg0, %arg0 : i32 + %1 = spirv.SLessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_slt +func.func @const_fold_scalar_slt() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThan %c5, %c6 : i32 + %1 = spirv.SLessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_slt +func.func @const_fold_vector_slt() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.SLessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @sle_same +func.func @sle_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.SLessThanEqual %arg0, %arg0 : i32 + %1 = spirv.SLessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_sle +func.func @const_fold_scalar_sle() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThanEqual %c5, %c6 : i32 + %1 = spirv.SLessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_sle +func.func @const_fold_vector_sle() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.SLessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ult_same +func.func @ult_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.ULessThan %arg0, %arg0 : i32 + %1 = spirv.ULessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ult +func.func @const_fold_scalar_ult() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThan %c5, %cn6 : i32 + %1 = spirv.ULessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ult +func.func @const_fold_vector_ult() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.ULessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ule_same +func.func @ule_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.ULessThanEqual %arg0, %arg0 : i32 + %1 = spirv.ULessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ule +func.func @const_fold_scalar_ule() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThanEqual %c5, %cn6 : i32 + %1 = spirv.ULessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ule +func.func @const_fold_vector_ule() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.ULessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.LeftShiftLogical //===----------------------------------------------------------------------===// -- GitLab From 7d2d8e2a7245e4e64da22cb3c422ea3be5a0bf0a Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Mon, 25 Mar 2024 10:59:42 +0900 Subject: [PATCH 284/933] [mlir][complex] Fastmath flag for the trigonometric ops in complex (#85563) Support Fastmath flag to convert trigonometric ops in the complex dialect. See: https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981 --- .../ComplexToStandard/ComplexToStandard.cpp | 50 +++++++++++-------- .../convert-to-standard.mlir | 46 +++++++++++++++++ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp index 76729278ec1b..17f64f1b65b7 100644 --- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp +++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp @@ -196,6 +196,7 @@ struct TrigonometricOpConversion : public OpConversionPattern { auto loc = op.getLoc(); auto type = cast(adaptor.getComplex().getType()); auto elementType = cast(type.getElementType()); + arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); Value real = rewriter.create(loc, elementType, adaptor.getComplex()); @@ -207,14 +208,14 @@ struct TrigonometricOpConversion : public OpConversionPattern { // implementation in the subclass to combine them. Value half = rewriter.create( loc, elementType, rewriter.getFloatAttr(elementType, 0.5)); - Value exp = rewriter.create(loc, imag); - Value scaledExp = rewriter.create(loc, half, exp); - Value reciprocalExp = rewriter.create(loc, half, exp); - Value sin = rewriter.create(loc, real); - Value cos = rewriter.create(loc, real); + Value exp = rewriter.create(loc, imag, fmf); + Value scaledExp = rewriter.create(loc, half, exp, fmf); + Value reciprocalExp = rewriter.create(loc, half, exp, fmf); + Value sin = rewriter.create(loc, real, fmf); + Value cos = rewriter.create(loc, real, fmf); auto resultPair = - combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter); + combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf); rewriter.replaceOpWithNewOp(op, type, resultPair.first, resultPair.second); @@ -223,15 +224,17 @@ struct TrigonometricOpConversion : public OpConversionPattern { virtual std::pair combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const = 0; + Value cos, ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const = 0; }; struct CosOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex cosine is defined as; // cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy))) // Plugging in: @@ -241,10 +244,12 @@ struct CosOpConversion : public TrigonometricOpConversion { // We get: // Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x // Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x - Value sum = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultReal = rewriter.create(loc, sum, cos); - Value diff = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultImag = rewriter.create(loc, diff, sin); + Value sum = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultReal = rewriter.create(loc, sum, cos, fmf); + Value diff = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultImag = rewriter.create(loc, diff, sin, fmf); return {resultReal, resultImag}; } }; @@ -813,9 +818,10 @@ struct NegOpConversion : public OpConversionPattern { struct SinOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex sine is defined as; // sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy))) // Plugging in: @@ -825,10 +831,12 @@ struct SinOpConversion : public TrigonometricOpConversion { // We get: // Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x // Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x - Value sum = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultReal = rewriter.create(loc, sum, sin); - Value diff = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultImag = rewriter.create(loc, diff, cos); + Value sum = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultReal = rewriter.create(loc, sum, sin, fmf); + Value diff = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultImag = rewriter.create(loc, diff, cos, fmf); return {resultReal, resultImag}; } }; diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir index 5918ff2e0f36..bac94aae6b74 100644 --- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir +++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir @@ -1834,3 +1834,49 @@ func.func @complex_sqrt_with_fmf(%arg: complex) -> complex { // CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32 // CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex // CHECK: return %[[VAR41]] : complex + +// ----- + +// CHECK-LABEL: func @complex_cos_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_cos_with_fmf(%arg: complex) -> complex { + %cos = complex.cos %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] + +// ----- + +// CHECK-LABEL: func @complex_sin_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_sin_with_fmf(%arg: complex) -> complex { + %cos = complex.sin %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] -- GitLab From 2e4e04c59043a645572cf548de5d9c333a5d6641 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Mon, 25 Mar 2024 10:06:12 +0800 Subject: [PATCH 285/933] [X86][BF16] Do not lower to VCVTNEPS2BF16 without AVX512VL (#86395) Fixes: #86305 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ++- llvm/test/CodeGen/X86/pr86305.ll | 74 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/X86/pr86305.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 35f756ea5e1d..9acbe17d0bca 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21512,7 +21512,9 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { } if (VT.getScalarType() == MVT::bf16) { - if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT)) + if (SVT.getScalarType() == MVT::f32 && + ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) return Op; return SDValue(); } @@ -21619,7 +21621,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op, SDLoc DL(Op); MVT SVT = Op.getOperand(0).getSimpleValueType(); - if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) { + if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) { SDValue Res; Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0)); Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res); diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll new file mode 100644 index 000000000000..79b42bb2532c --- /dev/null +++ b/llvm/test/CodeGen/X86/pr86305.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16 | FileCheck %s + +define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { +; CHECK-LABEL: add: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movq %rdx, %rbx +; CHECK-NEXT: movzwl (%rsi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm1 +; CHECK-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, (%rbx) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq + %a = load bfloat, ptr %pa + %b = load bfloat, ptr %pb + %add = fadd bfloat %a, %b + store bfloat %add, ptr %pc + ret void +} + +define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind { +; CHECK-LABEL: fptrunc_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %ebx +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %ebp +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r14d +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r15d +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %eax +; CHECK-NEXT: vmovd %r15d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r14d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq + %b = fptrunc <4 x float> %a to <4 x bfloat> + ret <4 x bfloat> %b +} -- GitLab From 5e5b6561029665e69e033cff4216fecb78302259 Mon Sep 17 00:00:00 2001 From: Sergei Barannikov Date: Mon, 25 Mar 2024 05:13:48 +0300 Subject: [PATCH 286/933] [MC] Make `MCParsedAsmOperand::getReg()` return `MCRegister` (#86444) --- .../llvm/MC/MCParser/MCParsedAsmOperand.h | 3 ++- .../llvm/MC/MCParser/MCTargetAsmParser.h | 4 +-- llvm/lib/MC/MCParser/MCTargetAsmParser.cpp | 6 +++++ .../AArch64/AsmParser/AArch64AsmParser.cpp | 2 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2 +- .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 2 +- .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 2 +- .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 2 +- .../Target/CSKY/AsmParser/CSKYAsmParser.cpp | 2 +- .../Hexagon/AsmParser/HexagonAsmParser.cpp | 2 +- .../Target/Lanai/AsmParser/LanaiAsmParser.cpp | 2 +- .../AsmParser/LoongArchAsmParser.cpp | 4 +-- .../Target/M68k/AsmParser/M68kAsmParser.cpp | 4 +-- .../MSP430/AsmParser/MSP430AsmParser.cpp | 2 +- .../Target/Mips/AsmParser/MipsAsmParser.cpp | 2 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 26 ++++++++++--------- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 +-- .../Target/Sparc/AsmParser/SparcAsmParser.cpp | 2 +- .../SystemZ/AsmParser/SystemZAsmParser.cpp | 2 +- llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp | 2 +- .../AsmParser/WebAssemblyAsmParser.cpp | 2 +- llvm/lib/Target/X86/AsmParser/X86Operand.h | 2 +- .../Xtensa/AsmParser/XtensaAsmParser.cpp | 2 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 2 +- 24 files changed, 46 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 0c9668904e82..27ecb7b85d22 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -15,6 +15,7 @@ namespace llvm { +class MCRegister; class raw_ostream; /// MCParsedAsmOperand - This abstract class represents a source-level assembly @@ -57,7 +58,7 @@ public: virtual bool isImm() const = 0; /// isReg - Is this a register operand? virtual bool isReg() const = 0; - virtual unsigned getReg() const = 0; + virtual MCRegister getReg() const = 0; /// isMem - Is this a memory operand? virtual bool isMem() const = 0; diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 7edd3f8ce490..49ce417e6fbb 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -514,9 +514,7 @@ public: /// by the tied-operands checks in the AsmMatcher. This method can be /// overridden to allow e.g. a sub- or super-register as the tied operand. virtual bool areEqualRegs(const MCParsedAsmOperand &Op1, - const MCParsedAsmOperand &Op2) const { - return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); - } + const MCParsedAsmOperand &Op2) const; // Return whether this parser uses assignment statements with equals tokens virtual bool equalIsAsmAssignment() { return true; }; diff --git a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp index 0db5fb36f795..665d92eb9a21 100644 --- a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp +++ b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp @@ -8,6 +8,7 @@ #include "llvm/MC/MCParser/MCTargetAsmParser.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCRegister.h" using namespace llvm; @@ -48,3 +49,8 @@ ParseStatus MCTargetAsmParser::parseDirective(AsmToken DirectiveID) { return ParseStatus::Failure; return ParseStatus::NoMatch; } + +bool MCTargetAsmParser::areEqualRegs(const MCParsedAsmOperand &Op1, + const MCParsedAsmOperand &Op2) const { + return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); +} diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index b807aaf76fdb..21643ebb4138 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -654,7 +654,7 @@ public: return Barrier.HasnXSModifier; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..4648df199c74 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -981,7 +981,7 @@ public: return Imm.Type; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isRegKind()); return Reg.RegNo; } diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 9cfdb15a0f43..2ad576ab1a9c 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -1002,7 +1002,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp index db4aa03437c6..383dfcc31117 100644 --- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp +++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp @@ -195,7 +195,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_Memri) && "Invalid access!"); return RegImm.Reg; diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 1688355f427c..9672ed009e9b 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -148,7 +148,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp index 4711e58bbed6..30bd3dcefa60 100644 --- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp +++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp @@ -400,7 +400,7 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index 864591d4eb95..092cccbcca9c 100644 --- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -245,7 +245,7 @@ public: /// getEndLoc - Get the location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp index ff3649b77e35..6ab1375b974e 100644 --- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp +++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp @@ -151,7 +151,7 @@ public: // getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isReg() && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp index cf163e4e1200..20284b18428b 100644 --- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp +++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp @@ -467,9 +467,9 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } const MCExpr *getImm() const { diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp index b2c0fda1ccc2..126176133dc0 100644 --- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp +++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp @@ -157,7 +157,7 @@ public: bool isDReg() const; bool isFPDReg() const; bool isFPCReg() const; - unsigned getReg() const override; + MCRegister getReg() const override; void addRegOperands(MCInst &Inst, unsigned N) const; static std::unique_ptr createMemOp(M68kMemOp MemOp, SMLoc Start, @@ -312,7 +312,7 @@ bool M68kOperand::isReg() const { return Kind == KindTy::MemOp && MemOp.Op == M68kMemOp::Kind::Reg; } -unsigned M68kOperand::getReg() const { +MCRegister M68kOperand::getReg() const { assert(isReg()); return MemOp.OuterReg; } diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp index 818a468612a5..2bc1a89ef59c 100644 --- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp +++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp @@ -183,7 +183,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Reg && "Invalid access!"); return Reg; } diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 9d6e8dc573a8..076e0a20cb97 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -1458,7 +1458,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { // As a special case until we sort out the definition of div/divu, accept // $0/$zero here so that MCK_ZERO works correctly. if (Kind == k_RegisterIndex && RegIdx.Index == 0 && diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index 8108cfa521c8..55978af38000 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -276,9 +276,11 @@ public: return TLSReg.Sym; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Not implemented"); } + + unsigned getRegNum() const { assert(isRegNumber() && "Invalid access!"); - return (unsigned) Imm.Val; + return (unsigned)Imm.Val; } unsigned getFpReg() const { @@ -459,22 +461,22 @@ public: void addRegGPRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegsNoR0[getRegNum()])); } void addRegG8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegs[getRegNum()])); } void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegsNoX0[getRegNum()])); } void addRegG8pRCOperands(MCInst &Inst, unsigned N) const { @@ -498,12 +500,12 @@ public: void addRegF4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegF8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegFpRCOperands(MCInst &Inst, unsigned N) const { @@ -513,12 +515,12 @@ public: void addRegVFRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VFRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VFRegs[getRegNum()])); } void addRegVRRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VRegs[getRegNum()])); } void addRegVSRCOperands(MCInst &Inst, unsigned N) const { @@ -538,12 +540,12 @@ public: void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegSPERCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(SPERegs[getReg()])); + Inst.addOperand(MCOperand::createReg(SPERegs[getRegNum()])); } void addRegACCRCOperands(MCInst &Inst, unsigned N) const { diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index cb2ba52390e2..5e594d6cad6b 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -977,9 +977,9 @@ public: return Imm.IsRV64; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } StringRef getSysReg() const { diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp index be4ec1e9dce2..67e2b9d7c997 100644 --- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp +++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp @@ -307,7 +307,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp index a58e8e0dfedf..f2c04215d12d 100644 --- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp +++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp @@ -227,7 +227,7 @@ public: bool isReg(RegisterKind RegKind) const { return Kind == KindReg && Reg.Kind == RegKind; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindReg && "Not a register"); return Reg.Num; } diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp index f9e30a3a9378..691fe8fe3aa4 100644 --- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp +++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp @@ -344,7 +344,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 3cc4d50271eb..020c0d6229d2 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -100,7 +100,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand { bool isReg() const override { return false; } bool isBrList() const { return Kind == BrList; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Assembly inspects a register operand"); return 0; } diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h index 641158cb351f..78669784dd03 100644 --- a/llvm/lib/Target/X86/AsmParser/X86Operand.h +++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h @@ -167,7 +167,7 @@ struct X86Operand final : public MCParsedAsmOperand { Tok.Length = Value.size(); } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNo; } diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp index 3f808298527f..1fa00af2111e 100644 --- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp +++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp @@ -244,7 +244,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 5df7990d8fc2..0a2b1cf0f9a6 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -2519,7 +2519,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) { // Check for register operands, including sub-classes. OS << " if (Operand.isReg()) {\n"; OS << " MatchClassKind OpKind;\n"; - OS << " switch (Operand.getReg()) {\n"; + OS << " switch (Operand.getReg().id()) {\n"; OS << " default: OpKind = InvalidMatchClass; break;\n"; for (const auto &RC : Info.RegisterClasses) OS << " case " << RC.first->getValueAsString("Namespace") -- GitLab From 6af6416e89de1f4656a145c4843226b468718434 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 11:44:16 +0800 Subject: [PATCH 287/933] [RISCV] Add a tune feature to disable stripping W suffix (#86255) We have a hidden option to disable it, but I'd like to make it a tune feature. For some implementations, instructions with W suffix would be less costly as they only perform on 32 bits data. Though we may lose some chances to compress. --- llvm/lib/Target/RISCV/RISCVFeatures.td | 4 ++ llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 2 +- llvm/test/CodeGen/RISCV/strip-w-suffix.ll | 74 +++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index f3e641e25018..6ef2289bb4be 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1226,6 +1226,10 @@ def TuneNoSinkSplatOperands "false", "Disable sink splat operands to enable .vx, .vf," ".wx, and .wf instructions">; +def TuneNoStripWSuffix + : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false", + "Disable strip W suffix">; + def TuneConditionalCompressedMoveFusion : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion", "true", "Enable branch+c.mv fusion">; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index dcf70e8cad64..39d420c2fbf0 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -672,7 +672,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII, const RISCVSubtarget &ST, MachineRegisterInfo &MRI) { - if (DisableStripWSuffix) + if (DisableStripWSuffix || !ST.enableStripWSuffix()) return false; bool MadeChange = false; diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll new file mode 100644 index 000000000000..4124b3d0d360 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=STRIP %s +; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=NO-STRIP %s + +define i32 @addiw(i32 %a) { +; STRIP-LABEL: addiw: +; STRIP: # %bb.0: +; STRIP-NEXT: lui a1, 1 +; STRIP-NEXT: addi a1, a1, -1 +; STRIP-NEXT: addw a0, a0, a1 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addiw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: lui a1, 1 +; NO-STRIP-NEXT: addiw a1, a1, -1 +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: ret + %ret = add i32 %a, 4095 + ret i32 %ret +} + +define i32 @addw(i32 %a, i32 %b) { +; STRIP-LABEL: addw: +; STRIP: # %bb.0: +; STRIP-NEXT: add a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %add = add i32 %a, %b + %ret = add i32 %add, 1024 + ret i32 %ret +} + +define i32 @mulw(i32 %a, i32 %b) { +; STRIP-LABEL: mulw: +; STRIP: # %bb.0: +; STRIP-NEXT: mul a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: mulw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: mulw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %mul = mul i32 %a, %b + %ret = add i32 %mul, 1024 + ret i32 %ret +} + +define i32 @slliw(i32 %a) { +; STRIP-LABEL: slliw: +; STRIP: # %bb.0: +; STRIP-NEXT: slli a0, a0, 1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: slliw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: slliw a0, a0, 1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %shl = shl i32 %a, 1 + %ret = add i32 %shl, 1024 + ret i32 %ret +} -- GitLab From d9746a6a5d523e21eee2c1b50c2f08aa19396965 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 12:42:59 +0800 Subject: [PATCH 288/933] [RISCV][NFC] Pass LMUL to copyPhysRegVector The opcode will be determined by LMUL. Reviewers: preames, lukel97, topperc Reviewed By: lukel97, topperc Pull Request: https://github.com/llvm/llvm-project/pull/84448 --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 52 ++++++++++++------------ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 4ca300f9151e..14c2d41e80f1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -299,36 +299,36 @@ void RISCVInstrInfo::copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF) const { + RISCVII::VLMUL LMul, unsigned NF) const { const TargetRegisterInfo *TRI = STI.getRegisterInfo(); - RISCVII::VLMUL LMul; + unsigned Opc; unsigned SubRegIdx; unsigned VVOpc, VIOpc; - switch (Opc) { + switch (LMul) { default: llvm_unreachable("Impossible LMUL for vector register copy."); - case RISCV::VMV1R_V: - LMul = RISCVII::LMUL_1; + case RISCVII::LMUL_1: + Opc = RISCV::VMV1R_V; SubRegIdx = RISCV::sub_vrm1_0; VVOpc = RISCV::PseudoVMV_V_V_M1; VIOpc = RISCV::PseudoVMV_V_I_M1; break; - case RISCV::VMV2R_V: - LMul = RISCVII::LMUL_2; + case RISCVII::LMUL_2: + Opc = RISCV::VMV2R_V; SubRegIdx = RISCV::sub_vrm2_0; VVOpc = RISCV::PseudoVMV_V_V_M2; VIOpc = RISCV::PseudoVMV_V_I_M2; break; - case RISCV::VMV4R_V: - LMul = RISCVII::LMUL_4; + case RISCVII::LMUL_4: + Opc = RISCV::VMV4R_V; SubRegIdx = RISCV::sub_vrm4_0; VVOpc = RISCV::PseudoVMV_V_V_M4; VIOpc = RISCV::PseudoVMV_V_I_M4; break; - case RISCV::VMV8R_V: + case RISCVII::LMUL_8: assert(NF == 1); - LMul = RISCVII::LMUL_8; + Opc = RISCV::VMV8R_V; SubRegIdx = RISCV::sub_vrm1_0; // There is no sub_vrm8_0. VVOpc = RISCV::PseudoVMV_V_V_M8; VIOpc = RISCV::PseudoVMV_V_I_M8; @@ -505,87 +505,87 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // VR->VR copies. if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1); return; } if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2); return; } if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4); return; } if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV8R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_8); return; } if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/2); return; } if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/2); return; } if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4, /*NF=*/2); return; } if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/3); return; } if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/3); return; } if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/4); return; } if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/4); return; } if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/5); return; } if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/6); return; } if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/7); return; } if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/8); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 8a312ee5e779..dd049fca0597 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -69,7 +69,7 @@ public: void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF = 1) const; + RISCVII::VLMUL LMul, unsigned NF = 1) const; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override; -- GitLab From 373e77b4c0ad9b0bf370f0e5a32a4100a5459d82 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 25 Mar 2024 13:08:56 +0800 Subject: [PATCH 289/933] [RISCV] Generalize (sub zext, zext) -> (sext (sub zext, zext)) to add (#86248) This generalizes the combine added in #82455 to other binary ops, beginning with adds in this patch. Because the two zext operands are always +ve when treated as signed, and we don't get any overflow since the add is carried out in at least N * 2 bits of the narrow type, the result of the add will always be +ve. So we can use a zext for the outer extend, unlike sub which may produce a -ve result from two +ve operands. Although we could still use sext for add, I plan to add support for other binary ops like mul in a later patch, but mul requires zext to be correct (because the maximum value will take up the full N * 2 bits). So I've opted to use zext here too for consistency. Alive2 proof: https://alive2.llvm.org/ce/z/PRNsUM --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 75 +++++++--- .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll | 32 ++--- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 128 ++++++++---------- 3 files changed, 125 insertions(+), 110 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5214595485ca..4ace50aa477a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12913,6 +12913,55 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); } +// add (zext, zext) -> zext (add (zext, zext)) +// sub (zext, zext) -> sext (sub (zext, zext)) +// +// where the sum of the extend widths match, and the the range of the bin op +// fits inside the width of the narrower bin op. (For profitability on rvv, we +// use a power of two for both inner and outer extend.) +// +// TODO: Extend this to other binary ops +static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) { + + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND) + return SDValue(); + if (!N0.hasOneUse() || !N1.hasOneUse()) + return SDValue(); + + SDValue Src0 = N0.getOperand(0); + SDValue Src1 = N1.getOperand(0); + EVT SrcVT = Src0.getValueType(); + if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) || + SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 || + SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); + EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); + + Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); + Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); + + // Src0 and Src1 are zero extended, so they're always positive if signed. + // + // sub can produce a negative from two positive operands, so it needs sign + // extended. Other nodes produce a positive from two positive operands, so + // zero extend instead. + unsigned OuterExtend = + N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + + return DAG.getNode( + OuterExtend, SDLoc(N), VT, + DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1)); +} + // Try to turn (add (xor bool, 1) -1) into (neg bool). static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); @@ -12950,6 +12999,8 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, return V; if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) return V; + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (add (select lhs, rhs, cc, 0, y), x) -> // (select lhs, rhs, cc, x, (add x, y)) @@ -13017,28 +13068,8 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, } } - // sub (zext, zext) -> sext (sub (zext, zext)) - // where the sum of the extend widths match, and the inner zexts - // add at least one bit. (For profitability on rvv, we use a - // power of two for both inner and outer extend.) - if (VT.isVector() && Subtarget.getTargetLowering()->isTypeLegal(VT) && - N0.getOpcode() == N1.getOpcode() && N0.getOpcode() == ISD::ZERO_EXTEND && - N0.hasOneUse() && N1.hasOneUse()) { - SDValue Src0 = N0.getOperand(0); - SDValue Src1 = N1.getOperand(0); - EVT SrcVT = Src0.getValueType(); - if (Subtarget.getTargetLowering()->isTypeLegal(SrcVT) && - SrcVT == Src1.getValueType() && SrcVT.getScalarSizeInBits() >= 8 && - SrcVT.getScalarSizeInBits() < VT.getScalarSizeInBits() / 2) { - LLVMContext &C = *DAG.getContext(); - EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); - EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); - Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); - Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); - return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, - DAG.getNode(ISD::SUB, SDLoc(N), NarrowVT, Src0, Src1)); - } - } + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (sub x, (select lhs, rhs, cc, 0, y)) -> // (select lhs, rhs, cc, x, (sub x, y)) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll index 57a72c639b33..bc0bf5dd76ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll @@ -385,12 +385,12 @@ define <32 x i64> @vwaddu_v32i64(ptr %x, ptr %y) nounwind { define <2 x i32> @vwaddu_v2i32_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -912,12 +912,12 @@ define <4 x i64> @crash(<4 x i16> %x, <4 x i16> %y) { define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -930,12 +930,12 @@ define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -948,12 +948,12 @@ define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i16(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: vle16.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i16>, ptr %x %b = load <2 x i16>, ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 66a7eea18be5..0a7051633a19 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -468,11 +468,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -555,10 +553,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -588,11 +586,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -675,10 +671,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -708,11 +704,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -795,10 +789,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -828,11 +822,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -915,10 +907,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -948,11 +940,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1035,10 +1025,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1068,11 +1058,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1155,10 +1143,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1188,11 +1176,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1275,10 +1261,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1308,11 +1294,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From babbdad15b8049a6a78087d15a163d897f07d320 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:23:40 +0100 Subject: [PATCH 290/933] [AMDGPU] Handle non-register operands for S_SUB/ADD_U64_PSEUDO (#86104) This pseudo uses SSrc_b64 so it allows both an immediate or a register, but the lowering crashed on immediate operands. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- .../CodeGen/AMDGPU/add_sub_u64_pseudos.mir | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 7f0cff72c186..d437f339a687 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4859,8 +4859,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( if (Subtarget->hasScalarAddSub64()) { unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) - .addReg(Src0.getReg()) - .addReg(Src1.getReg()); + .add(Src0) + .add(Src1); } else { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const TargetRegisterClass *BoolRC = TRI->getBoolRC(); diff --git a/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir new file mode 100644 index 000000000000..cba114c3568a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir @@ -0,0 +1,68 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX11 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX12 %s + +--- +name: reg_ops +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: reg_ops + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub0 + ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY2]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], [[COPY3]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: reg_ops + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], [[DEF1]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = IMPLICIT_DEF + %2:sreg_64 = S_ADD_U64_PSEUDO %0, %1, implicit-def $scc +... + +--- +name: lhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: lhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 6565, [[COPY]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 0, [[COPY1]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: lhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 6565, [[DEF]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO 6565, %0, implicit-def $scc +... + +--- +name: rhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: rhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 6565, implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], 0, implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: rhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], 6565 + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO %0, 6565, implicit-def $scc +... -- GitLab From fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:40:35 +0100 Subject: [PATCH 291/933] [RFC][TableGen] Restructure TableGen Source (#80847) Refactor of the llvm-tblgen source into: - a "Basic" library, which contains the bare minimum utilities to build `llvm-min-tablegen` - a "Common" library which contains all of the helpers for TableGen backends. Such helpers can be shared by more than one backend, and even unit tested (e.g. CodeExpander is, maybe we can add more over time) Fixes #80647 --- llvm/unittests/TableGen/CMakeLists.txt | 2 +- llvm/unittests/TableGen/CodeExpanderTest.cpp | 4 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/AsmWriterEmitter.cpp | 14 ++-- llvm/utils/TableGen/Basic/CMakeLists.txt | 21 ++++++ .../{ => Basic}/CodeGenIntrinsics.cpp | 0 .../TableGen/{ => Basic}/CodeGenIntrinsics.h | 0 .../TableGen/{ => Basic}/SDNodeProperties.cpp | 0 .../TableGen/{ => Basic}/SDNodeProperties.h | 0 .../{ => Basic}/SequenceToOffsetTable.h | 0 llvm/utils/TableGen/CMakeLists.txt | 67 ++++++++----------- llvm/utils/TableGen/CallingConvEmitter.cpp | 2 +- llvm/utils/TableGen/CodeEmitterGen.cpp | 10 +-- llvm/utils/TableGen/CodeGenMapTable.cpp | 4 +- .../TableGen/{ => Common}/AsmWriterInst.cpp | 0 .../TableGen/{ => Common}/AsmWriterInst.h | 0 llvm/utils/TableGen/Common/CMakeLists.txt | 48 +++++++++++++ .../{ => Common}/CodeGenDAGPatterns.cpp | 0 .../{ => Common}/CodeGenDAGPatterns.h | 4 +- .../TableGen/{ => Common}/CodeGenHwModes.cpp | 0 .../TableGen/{ => Common}/CodeGenHwModes.h | 0 .../{ => Common}/CodeGenInstAlias.cpp | 0 .../TableGen/{ => Common}/CodeGenInstAlias.h | 0 .../{ => Common}/CodeGenInstruction.cpp | 0 .../{ => Common}/CodeGenInstruction.h | 0 .../{ => Common}/CodeGenRegisters.cpp | 0 .../TableGen/{ => Common}/CodeGenRegisters.h | 0 .../TableGen/{ => Common}/CodeGenSchedule.cpp | 0 .../TableGen/{ => Common}/CodeGenSchedule.h | 0 .../TableGen/{ => Common}/CodeGenTarget.cpp | 0 .../TableGen/{ => Common}/CodeGenTarget.h | 2 +- .../TableGen/{ => Common}/DAGISelMatcher.cpp | 0 .../TableGen/{ => Common}/DAGISelMatcher.h | 0 .../{ => Common}/GlobalISel/CXXPredicates.cpp | 0 .../{ => Common}/GlobalISel/CXXPredicates.h | 0 .../{ => Common}/GlobalISel/CodeExpander.cpp | 0 .../{ => Common}/GlobalISel/CodeExpander.h | 0 .../{ => Common}/GlobalISel/CodeExpansions.h | 0 .../{ => Common}/GlobalISel/CombinerUtils.h | 0 .../GlobalISel}/GlobalISelMatchTable.cpp | 4 +- .../GlobalISel}/GlobalISelMatchTable.h | 2 +- .../GlobalISelMatchTableExecutorEmitter.cpp | 0 .../GlobalISelMatchTableExecutorEmitter.h | 2 +- .../{ => Common}/GlobalISel/MatchDataInfo.cpp | 0 .../{ => Common}/GlobalISel/MatchDataInfo.h | 0 .../{ => Common}/GlobalISel/Patterns.cpp | 4 +- .../{ => Common}/GlobalISel/Patterns.h | 0 .../TableGen/{ => Common}/InfoByHwMode.cpp | 0 .../TableGen/{ => Common}/InfoByHwMode.h | 0 .../TableGen/{ => Common}/OptEmitter.cpp | 0 llvm/utils/TableGen/{ => Common}/OptEmitter.h | 0 .../{ => Common}/PredicateExpander.cpp | 0 .../TableGen/{ => Common}/PredicateExpander.h | 0 .../{ => Common}/SubtargetFeatureInfo.cpp | 0 .../{ => Common}/SubtargetFeatureInfo.h | 0 llvm/utils/TableGen/{ => Common}/Types.cpp | 0 llvm/utils/TableGen/{ => Common}/Types.h | 0 .../{ => Common}/VarLenCodeEmitterGen.cpp | 15 ++--- .../{ => Common}/VarLenCodeEmitterGen.h | 0 llvm/utils/TableGen/CompressInstEmitter.cpp | 6 +- llvm/utils/TableGen/DAGISelEmitter.cpp | 8 +-- llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/DAGISelMatcherGen.cpp | 14 ++-- llvm/utils/TableGen/DAGISelMatcherOpt.cpp | 6 +- llvm/utils/TableGen/DFAEmitter.cpp | 2 +- llvm/utils/TableGen/DFAPacketizerEmitter.cpp | 4 +- llvm/utils/TableGen/DXILEmitter.cpp | 4 +- llvm/utils/TableGen/DecoderEmitter.cpp | 10 +-- llvm/utils/TableGen/DisassemblerEmitter.cpp | 2 +- llvm/utils/TableGen/FastISelEmitter.cpp | 10 +-- llvm/utils/TableGen/GlobalISel/CMakeLists.txt | 20 ------ .../TableGen/GlobalISelCombinerEmitter.cpp | 24 +++---- llvm/utils/TableGen/GlobalISelEmitter.cpp | 18 ++--- llvm/utils/TableGen/InstrDocsEmitter.cpp | 6 +- llvm/utils/TableGen/InstrInfoEmitter.cpp | 16 ++--- llvm/utils/TableGen/IntrinsicEmitter.cpp | 4 +- .../TableGen/MacroFusionPredicatorEmitter.cpp | 4 +- llvm/utils/TableGen/OptParserEmitter.cpp | 2 +- llvm/utils/TableGen/OptRSTEmitter.cpp | 2 +- llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 4 +- llvm/utils/TableGen/RegisterBankEmitter.cpp | 6 +- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 12 ++-- .../utils/TableGen/SearchableTableEmitter.cpp | 4 +- llvm/utils/TableGen/SubtargetEmitter.cpp | 8 +-- .../WebAssemblyDisassemblerEmitter.cpp | 2 +- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86MnemonicTables.cpp | 4 +- llvm/utils/TableGen/X86RecognizableInstr.h | 2 +- 89 files changed, 238 insertions(+), 203 deletions(-) create mode 100644 llvm/utils/TableGen/Basic/CMakeLists.txt rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.h (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.h (100%) rename llvm/utils/TableGen/{ => Basic}/SequenceToOffsetTable.h (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.cpp (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.h (100%) create mode 100644 llvm/utils/TableGen/Common/CMakeLists.txt rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.h (99%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.h (99%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.cpp (100%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpansions.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CombinerUtils.h (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.cpp (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.h (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.h (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.cpp (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.h (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.cpp (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.h (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.h (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/Types.cpp (100%) rename llvm/utils/TableGen/{ => Common}/Types.h (100%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.cpp (97%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.h (100%) delete mode 100644 llvm/utils/TableGen/GlobalISel/CMakeLists.txt diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt index 7830e0218045..fae0eee06e4b 100644 --- a/llvm/unittests/TableGen/CMakeLists.txt +++ b/llvm/unittests/TableGen/CMakeLists.txt @@ -15,4 +15,4 @@ add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB ParserEntryPointTest.cpp ) -target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen) +target_link_libraries(TableGenTests PRIVATE LLVMTableGenCommon LLVMTableGen) diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp index 4a9a0e8c114b..1528884ffdf6 100644 --- a/llvm/unittests/TableGen/CodeExpanderTest.cpp +++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 0a2b1cf0f9a6..8b82ce899a48 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -95,12 +95,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SubtargetFeatureInfo.h" -#include "Types.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp index a27061ee585a..16661cd29edc 100644 --- a/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -11,13 +11,13 @@ // //===----------------------------------------------------------------------===// -#include "AsmWriterInst.h" -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/AsmWriterInst.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt new file mode 100644 index 000000000000..f2927d05c175 --- /dev/null +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -0,0 +1,21 @@ +# The basic TableGen library contains as little dependencies as possible. +# In particular, it does not depend on vt_gen -> it does not use ValueTypes. +# +# This library is the only thing included in `llvm-min-tablegen`. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL + CodeGenIntrinsics.cpp + SDNodeProperties.cpp +) +set_target_properties(LLVMTableGenBasic PROPERTIES FOLDER "Tablegenning") + +# Users may include its headers as "Basic/*.h" +target_include_directories(LLVMTableGenBasic + INTERFACE + $ + ) diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.cpp rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.h rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.h diff --git a/llvm/utils/TableGen/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.cpp rename to llvm/utils/TableGen/Basic/SDNodeProperties.cpp diff --git a/llvm/utils/TableGen/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.h rename to llvm/utils/TableGen/Basic/SDNodeProperties.h diff --git a/llvm/utils/TableGen/SequenceToOffsetTable.h b/llvm/utils/TableGen/Basic/SequenceToOffsetTable.h similarity index 100% rename from llvm/utils/TableGen/SequenceToOffsetTable.h rename to llvm/utils/TableGen/Basic/SequenceToOffsetTable.h diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 0100bf345ec2..14690329cabf 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -1,26 +1,25 @@ -add_subdirectory(GlobalISel) +# Basic utilities which is the strict minimum needed to build +# llvm-min-tblgen. +add_subdirectory(Basic) +# Common utilities are all of the reusable components and helper +# code needed by the backends. +add_subdirectory(Common) -add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL +set(LLVM_LINK_COMPONENTS Support) + +# llvm-min-tablegen only contains a subset of backends necessary to +# build llvm/include. It must not depend on TableGenCommon, as +# TableGenCommon depends on this already to generate things such as +# ValueType definitions. +add_tablegen(llvm-min-tblgen LLVM_HEADERS + TableGen.cpp Attributes.cpp - CodeGenIntrinsics.cpp DirectiveEmitter.cpp IntrinsicEmitter.cpp RISCVTargetDefEmitter.cpp - SDNodeProperties.cpp VTEmitter.cpp - PARTIAL_SOURCES_INTENDED - - LINK_COMPONENTS - Support - TableGen - ) -set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") + $ -set(LLVM_LINK_COMPONENTS Support) - -add_tablegen(llvm-min-tblgen LLVM_HEADERS - TableGen.cpp - $ PARTIAL_SOURCES_INTENDED ) set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning") @@ -35,63 +34,51 @@ add_tablegen(llvm-tblgen LLVM EXPORT LLVM AsmMatcherEmitter.cpp AsmWriterEmitter.cpp - AsmWriterInst.cpp - CTagsEmitter.cpp + Attributes.cpp CallingConvEmitter.cpp CodeEmitterGen.cpp - CodeGenDAGPatterns.cpp - CodeGenHwModes.cpp - CodeGenInstAlias.cpp - CodeGenInstruction.cpp CodeGenMapTable.cpp - CodeGenRegisters.cpp - CodeGenSchedule.cpp - CodeGenTarget.cpp + CompressInstEmitter.cpp + CTagsEmitter.cpp DAGISelEmitter.cpp DAGISelMatcherEmitter.cpp DAGISelMatcherGen.cpp DAGISelMatcherOpt.cpp - DAGISelMatcher.cpp DecoderEmitter.cpp DFAEmitter.cpp DFAPacketizerEmitter.cpp + DirectiveEmitter.cpp DisassemblerEmitter.cpp DXILEmitter.cpp ExegesisEmitter.cpp FastISelEmitter.cpp GlobalISelCombinerEmitter.cpp GlobalISelEmitter.cpp - GlobalISelMatchTable.cpp - GlobalISelMatchTableExecutorEmitter.cpp - InfoByHwMode.cpp - InstrInfoEmitter.cpp InstrDocsEmitter.cpp - OptEmitter.cpp + InstrInfoEmitter.cpp + IntrinsicEmitter.cpp + MacroFusionPredicatorEmitter.cpp OptParserEmitter.cpp OptRSTEmitter.cpp - PredicateExpander.cpp PseudoLoweringEmitter.cpp - CompressInstEmitter.cpp - MacroFusionPredicatorEmitter.cpp RegisterBankEmitter.cpp RegisterInfoEmitter.cpp + RISCVTargetDefEmitter.cpp SearchableTableEmitter.cpp SubtargetEmitter.cpp - SubtargetFeatureInfo.cpp TableGen.cpp - Types.cpp - VarLenCodeEmitterGen.cpp - X86DisassemblerTables.cpp + VTEmitter.cpp + WebAssemblyDisassemblerEmitter.cpp X86CompressEVEXTablesEmitter.cpp + X86DisassemblerTables.cpp X86FoldTablesEmitter.cpp X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp - WebAssemblyDisassemblerEmitter.cpp $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel) +target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp index 3c3a2874ce80..ec6ef56a66fa 100644 --- a/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp index 9194c13ccdcb..a57885f22d7e 100644 --- a/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -22,11 +22,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "VarLenCodeEmitterGen.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp index 03af0b49ba97..fbf1d47c0327 100644 --- a/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -75,8 +75,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" using namespace llvm; diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/Common/AsmWriterInst.cpp similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.cpp rename to llvm/utils/TableGen/Common/AsmWriterInst.cpp diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/Common/AsmWriterInst.h similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.h rename to llvm/utils/TableGen/Common/AsmWriterInst.h diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt new file mode 100644 index 000000000000..491d9bd2949d --- /dev/null +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -0,0 +1,48 @@ +# The common library is similar to the basic library except it can +# depend on vt_gen. +# +# This library contains the bulk of the supporting code for all +# TableGen backends. It's split off as a separate library to +# allow unit-testing those components. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL + GlobalISel/CodeExpander.cpp + GlobalISel/CXXPredicates.cpp + GlobalISel/GlobalISelMatchTable.cpp + GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp + GlobalISel/MatchDataInfo.cpp + GlobalISel/Patterns.cpp + + AsmWriterInst.cpp + CodeGenDAGPatterns.cpp + CodeGenHwModes.cpp + CodeGenInstAlias.cpp + CodeGenInstruction.cpp + CodeGenRegisters.cpp + CodeGenSchedule.cpp + CodeGenTarget.cpp + DAGISelMatcher.cpp + InfoByHwMode.cpp + OptEmitter.cpp + PredicateExpander.cpp + SubtargetFeatureInfo.cpp + Types.cpp + VarLenCodeEmitterGen.cpp + $ + + DEPENDS + vt_gen + ) +set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") +target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) + +# Users may include its headers as "Common/*.h" +target_include_directories(LLVMTableGenCommon + PUBLIC + $ + ) diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenDAGPatterns.cpp rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h similarity index 99% rename from llvm/utils/TableGen/CodeGenDAGPatterns.h rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.h index 823c40c922cb..7fcd39a9e940 100644 --- a/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h @@ -14,9 +14,9 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H -#include "CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SDNodeProperties.h" #include "CodeGenTarget.h" -#include "SDNodeProperties.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.cpp rename to llvm/utils/TableGen/Common/CodeGenHwModes.cpp diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/Common/CodeGenHwModes.h similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.h rename to llvm/utils/TableGen/Common/CodeGenHwModes.h diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.cpp rename to llvm/utils/TableGen/Common/CodeGenInstAlias.cpp diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/Common/CodeGenInstAlias.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.h rename to llvm/utils/TableGen/Common/CodeGenInstAlias.h diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.cpp rename to llvm/utils/TableGen/Common/CodeGenInstruction.cpp diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.h rename to llvm/utils/TableGen/Common/CodeGenInstruction.h diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.cpp rename to llvm/utils/TableGen/Common/CodeGenRegisters.cpp diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.h rename to llvm/utils/TableGen/Common/CodeGenRegisters.h diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.cpp rename to llvm/utils/TableGen/Common/CodeGenSchedule.cpp diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.h rename to llvm/utils/TableGen/Common/CodeGenSchedule.h diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenTarget.cpp rename to llvm/utils/TableGen/Common/CodeGenTarget.cpp diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h similarity index 99% rename from llvm/utils/TableGen/CodeGenTarget.h rename to llvm/utils/TableGen/Common/CodeGenTarget.h index e109c717dc01..df4c22ebb379 100644 --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/Common/CodeGenTarget.h @@ -16,10 +16,10 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#include "Basic/SDNodeProperties.h" #include "CodeGenHwModes.h" #include "CodeGenInstruction.h" #include "InfoByHwMode.h" -#include "SDNodeProperties.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.cpp rename to llvm/utils/TableGen/Common/DAGISelMatcher.cpp diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.h rename to llvm/utils/TableGen/Common/DAGISelMatcher.h diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.h rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpansions.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CombinerUtils.h rename to llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 45fb41b89f27..193f95443b16 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include "GlobalISelMatchTable.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" #include "llvm/Support/LEB128.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index b1ab7da8db44..aa86fad763d1 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H -#include "CodeGenDAGPatterns.h" +#include "Common/CodeGenDAGPatterns.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h index 7e952d6df309..d2b6a74c7577 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h @@ -15,7 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H -#include "SubtargetFeatureInfo.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.h rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISel/Patterns.cpp rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp index 758eac2dfebd..388bf7e9e833 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp @@ -7,11 +7,11 @@ //===----------------------------------------------------------------------===// #include "Patterns.h" -#include "../CodeGenInstruction.h" -#include "../CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" #include "CXXPredicates.h" #include "CodeExpander.h" #include "CodeExpansions.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/Patterns.h rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.h diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.cpp rename to llvm/utils/TableGen/Common/InfoByHwMode.cpp diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.h rename to llvm/utils/TableGen/Common/InfoByHwMode.h diff --git a/llvm/utils/TableGen/OptEmitter.cpp b/llvm/utils/TableGen/Common/OptEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/OptEmitter.cpp rename to llvm/utils/TableGen/Common/OptEmitter.cpp diff --git a/llvm/utils/TableGen/OptEmitter.h b/llvm/utils/TableGen/Common/OptEmitter.h similarity index 100% rename from llvm/utils/TableGen/OptEmitter.h rename to llvm/utils/TableGen/Common/OptEmitter.h diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.cpp rename to llvm/utils/TableGen/Common/PredicateExpander.cpp diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/Common/PredicateExpander.h similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.h rename to llvm/utils/TableGen/Common/PredicateExpander.h diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.cpp rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.h similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.h rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.h diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Common/Types.cpp similarity index 100% rename from llvm/utils/TableGen/Types.cpp rename to llvm/utils/TableGen/Common/Types.cpp diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Common/Types.h similarity index 100% rename from llvm/utils/TableGen/Types.h rename to llvm/utils/TableGen/Common/Types.h diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp similarity index 97% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.cpp rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp index bfb7e5c33317..4263d8f41715 100644 --- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp +++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp @@ -337,8 +337,8 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits, return; } - IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " - << "/*Index*/" << Index << "},"; + IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " << "/*Index*/" + << Index << "},"; SS.indent(4); for (unsigned I = 0; I < Bits.getNumWords(); ++I, ++Index) @@ -371,8 +371,8 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( if (ModeIt == InstIt->second.end()) ModeIt = InstIt->second.find(Universal); if (ModeIt == InstIt->second.end()) { - IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" - << "// " << R->getName() << " no encoding\n"; + IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" << "// " << R->getName() + << " no encoding\n"; continue; } const VarLenInst &VLI = ModeIt->second; @@ -492,10 +492,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n"; - SS.indent(I) << "Inst.insertBits(" - << "Scratch.extractBits(" << utostr(NumBits) << ", " - << utostr(LoBit) << ")" - << ", " << Offset << ");\n"; + SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits(" + << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", " + << Offset << ");\n"; HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit); } diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h similarity index 100% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.h rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp index f703fff0ef3e..fcf77934faac 100644 --- a/llvm/utils/TableGen/CompressInstEmitter.cpp +++ b/llvm/utils/TableGen/CompressInstEmitter.cpp @@ -64,9 +64,9 @@ //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp index 336cee09b90c..b43a8e659dd9 100644 --- a/llvm/utils/TableGen/DAGISelEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 533b8c423690..dcecac4380ce 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -10,12 +10,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index e8bdabaa0c7e..99babdf07316 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "InfoByHwMode.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 047d285f9914..224102e49d98 100644 --- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp index ce8cc2a078d7..567184d3d5ee 100644 --- a/llvm/utils/TableGen/DFAEmitter.cpp +++ b/llvm/utils/TableGen/DFAEmitter.cpp @@ -21,7 +21,7 @@ //===----------------------------------------------------------------------===// #include "DFAEmitter.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/UniqueVector.h" diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp index 26ea1846ffae..3c74df048660 100644 --- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp +++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -14,8 +14,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" #include "DFAEmitter.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index af1efb8aa99f..47c89df35e19 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 732f34ed04c5..494dc93faace 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "TableGenBackends.h" -#include "VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/CachedHashString.h" diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp index 2d653af4d302..d41750075b41 100644 --- a/llvm/utils/TableGen/DisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "TableGenBackends.h" #include "WebAssemblyDisassemblerEmitter.h" #include "X86DisassemblerTables.h" diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp index f04c6e3b3bf0..acfdc20316b7 100644 --- a/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/llvm/utils/TableGen/FastISelEmitter.cpp @@ -16,11 +16,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt deleted file mode 100644 index 7262c4058399..000000000000 --- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(LLVM_LINK_COMPONENTS - Support - TableGen - ) - -add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB - CodeExpander.cpp - CXXPredicates.cpp - MatchDataInfo.cpp - Patterns.cpp - - DEPENDS - vt_gen - ) - -# Users may include its headers as "GlobalISel/*.h" -target_include_directories(LLVMTableGenGlobalISel - INTERFACE - $ - ) diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index dee3cb4d71a4..39b9f8a2ae17 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -26,18 +26,18 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" -#include "GlobalISel/CXXPredicates.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISel/MatchDataInfo.h" -#include "GlobalISel/Patterns.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/CXXPredicates.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CombinerUtils.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/GlobalISel/MatchDataInfo.h" +#include "Common/GlobalISel/Patterns.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/Hashing.h" diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index e86057422cd7..25e302ce1ca4 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -30,15 +30,15 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "InfoByHwMode.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/InfoByHwMode.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGenTypes/LowLevelType.h" #include "llvm/CodeGenTypes/MachineValueType.h" diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp index efabf6bb7ba6..f948540e18db 100644 --- a/llvm/utils/TableGen/InstrDocsEmitter.cpp +++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -18,9 +18,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp index 2d08447429d9..36f8fa146539 100644 --- a/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -11,15 +11,15 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" -#include "SequenceToOffsetTable.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "TableGenBackends.h" -#include "Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 50a34eac7ca3..a7e99fa4c050 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "SequenceToOffsetTable.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index 91c3b0b4359c..e9e63fa8d0de 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -38,8 +38,8 @@ // //===---------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp index c25f6c59cab3..6334af53f88f 100644 --- a/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/llvm/utils/TableGen/OptParserEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp index 5a7f079dc168..75b7cbdf2988 100644 --- a/llvm/utils/TableGen/OptRSTEmitter.cpp +++ b/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp index 7f692f29192d..01cfd4a1d982 100644 --- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp +++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp index 8b59411c5bc3..5546e727af38 100644 --- a/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/BitVector.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index c4fc1930488c..a1259bff6ba8 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp index 51f18f360ed3..48ee23db957d 100644 --- a/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp index d350d7de139f..2e2c57b802ee 100644 --- a/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp index 928129f24fcb..e9436ab16e44 100644 --- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -14,7 +14,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyDisassemblerEmitter.h" -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 0a9abbfe186e..c721502a395f 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -11,8 +11,8 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 1319042e48d0..5871e678b16e 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FormattedStream.h" diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp index aeafee157462..d9ceed40f7c7 100644 --- a/llvm/utils/TableGen/X86MnemonicTables.cpp +++ b/llvm/utils/TableGen/X86MnemonicTables.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 68af68fb5aa0..12fb41750cb3 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/Support/X86DisassemblerDecoderCommon.h" #include #include -- GitLab From aa962d67ee896f416e285a9298e45fc08ff95eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:42:41 +0100 Subject: [PATCH 292/933] [clang-repl] Fix Value for platforms where unqualified char is unsigned (#86118) Signedness of unqualified `char` is unspecified and varies between platforms. This patch adds `Char_U` in `REPL_BUILTIN_TYPES` to account for platforms that default to `unsigned char`. --- clang/include/clang/Interpreter/Value.h | 1 + clang/unittests/Interpreter/InterpreterTest.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h index c380cd91550d..d70e8f871902 100644 --- a/clang/include/clang/Interpreter/Value.h +++ b/clang/include/clang/Interpreter/Value.h @@ -76,6 +76,7 @@ class QualType; X(bool, Bool) \ X(char, Char_S) \ X(signed char, SChar) \ + X(unsigned char, Char_U) \ X(unsigned char, UChar) \ X(short, Short) \ X(unsigned short, UShort) \ diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index e76c0677db5e..69bc2da24288 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -340,6 +340,12 @@ TEST(InterpreterTest, Value) { EXPECT_EQ(V1.getKind(), Value::K_Int); EXPECT_FALSE(V1.isManuallyAlloc()); + Value V1b; + llvm::cantFail(Interp->ParseAndExecute("char c = 42;")); + llvm::cantFail(Interp->ParseAndExecute("c", &V1b)); + EXPECT_TRUE(V1b.getKind() == Value::K_Char_S || + V1b.getKind() == Value::K_Char_U); + Value V2; llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;")); llvm::cantFail(Interp->ParseAndExecute("y", &V2)); -- GitLab From 0cf4788d9d0df60980cb48d28aafe7a86aa15a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:44:25 +0100 Subject: [PATCH 293/933] [clang-repl] Factor out CreateJITBuilder() and allow specialization in derived classes (#84461) The LLJITBuilder interface provides a very convenient way to configure the ORCv2 JIT engine. IncrementalExecutor already used it internally to construct the JIT, but didn't provide external access. This patch lifts control of the creation process to the Interpreter and allows injection of a custom instance through the extended interface. The Interpreter's default behavior remains unchanged and the IncrementalExecutor remains an implementation detail. --- clang/include/clang/Interpreter/Interpreter.h | 8 ++ clang/lib/Interpreter/IncrementalExecutor.cpp | 33 ++--- clang/lib/Interpreter/IncrementalExecutor.h | 9 +- clang/lib/Interpreter/Interpreter.cpp | 26 +++- .../Interpreter/InterpreterExtensionsTest.cpp | 121 +++++++++++++++++- 5 files changed, 175 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 1dcba1ef9679..970e0245417b 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -30,6 +30,7 @@ namespace llvm { namespace orc { class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -127,6 +128,13 @@ protected: // custom runtime. virtual std::unique_ptr FindRuntimeInterface(); + // Lazily construct thev ORCv2 JITBuilder. This called when the internal + // IncrementalExecutor is created. The default implementation populates an + // in-process JIT with debugging support. Override this to configure the JIT + // engine used for execution. + virtual llvm::Expected> + CreateJITBuilder(CompilerInstance &CI); + public: virtual ~Interpreter(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 40bcef94797d..6f036107c14a 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" @@ -36,26 +37,28 @@ LLVM_ATTRIBUTE_USED void linkComponents() { namespace clang { +llvm::Expected> +IncrementalExecutor::createDefaultJITBuilder( + llvm::orc::JITTargetMachineBuilder JTMB) { + auto JITBuilder = std::make_unique(); + JITBuilder->setJITTargetMachineBuilder(std::move(JTMB)); + JITBuilder->setPrePlatformSetup([](llvm::orc::LLJIT &J) { + // Try to enable debugging of JIT'd code (only works with JITLink for + // ELF and MachO). + consumeError(llvm::orc::enableDebuggerSupport(J)); + return llvm::Error::success(); + }); + return std::move(JITBuilder); +} + IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, - llvm::Error &Err, - const clang::TargetInfo &TI) + llvm::orc::LLJITBuilder &JITBuilder, + llvm::Error &Err) : TSCtx(TSC) { using namespace llvm::orc; llvm::ErrorAsOutParameter EAO(&Err); - auto JTMB = JITTargetMachineBuilder(TI.getTriple()); - JTMB.addFeatures(TI.getTargetOpts().Features); - LLJITBuilder Builder; - Builder.setJITTargetMachineBuilder(JTMB); - Builder.setPrePlatformSetup( - [](LLJIT &J) { - // Try to enable debugging of JIT'd code (only works with JITLink for - // ELF and MachO). - consumeError(enableDebuggerSupport(J)); - return llvm::Error::success(); - }); - - if (auto JitOrErr = Builder.create()) + if (auto JitOrErr = JITBuilder.create()) Jit = std::move(*JitOrErr); else { Err = JitOrErr.takeError(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index dd0a210a0614..b4347209e14f 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -23,7 +23,9 @@ namespace llvm { class Error; namespace orc { +class JITTargetMachineBuilder; class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -44,8 +46,8 @@ class IncrementalExecutor { public: enum SymbolNameKind { IRName, LinkerName }; - IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err, - const clang::TargetInfo &TI); + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); ~IncrementalExecutor(); llvm::Error addModule(PartialTranslationUnit &PTU); @@ -56,6 +58,9 @@ public: getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const; llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + + static llvm::Expected> + createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); }; } // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 7fa52f2f15fc..cf31456b6950 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -372,15 +372,35 @@ Interpreter::Parse(llvm::StringRef Code) { return IncrParser->Parse(Code); } +static llvm::Expected +createJITTargetMachineBuilder(const std::string &TT) { + if (TT == llvm::sys::getProcessTriple()) + // This fails immediately if the target backend is not registered + return llvm::orc::JITTargetMachineBuilder::detectHost(); + + // If the target backend is not registered, LLJITBuilder::create() will fail + return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); +} + +llvm::Expected> +Interpreter::CreateJITBuilder(CompilerInstance &CI) { + auto JTMB = createJITTargetMachineBuilder(CI.getTargetOpts().Triple); + if (!JTMB) + return JTMB.takeError(); + return IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); +} + llvm::Error Interpreter::CreateExecutor() { - const clang::TargetInfo &TI = - getCompilerInstance()->getASTContext().getTargetInfo(); if (IncrExecutor) return llvm::make_error("Operation failed. " "Execution engine exists", std::error_code()); + llvm::Expected> JB = + CreateJITBuilder(*getCompilerInstance()); + if (!JB) + return JB.takeError(); llvm::Error Err = llvm::Error::success(); - auto Executor = std::make_unique(*TSCtx, Err, TI); + auto Executor = std::make_unique(*TSCtx, **JB, Err); if (!Err) IncrExecutor = std::move(Executor); diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index b7708616fd24..8bc429d9ec2d 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -18,14 +18,21 @@ #include "clang/Sema/Sema.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/Support/Error.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Threading.h" #include "llvm/Testing/Support/Error.h" #include "gmock/gmock.h" #include "gtest/gtest.h" + #include +#if defined(_AIX) +#define CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +#endif + using namespace clang; namespace { @@ -41,6 +48,10 @@ struct LLVMInitRAII { LLVMInitRAII() { llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmPrinter(); + LLVMInitializeARMTarget(); + LLVMInitializeARMTargetInfo(); + LLVMInitializeARMTargetMC(); + LLVMInitializeARMAsmPrinter(); } ~LLVMInitRAII() { llvm::llvm_shutdown(); } } LLVMInit; @@ -51,12 +62,30 @@ public: llvm::Error &Err) : Interpreter(std::move(CI), Err) {} - llvm::Error testCreateExecutor() { return Interpreter::CreateExecutor(); } + llvm::Error testCreateJITBuilderError() { + JB = nullptr; + return Interpreter::CreateExecutor(); + } + + llvm::Error testCreateExecutor() { + JB = std::make_unique(); + return Interpreter::CreateExecutor(); + } void resetExecutor() { Interpreter::ResetExecutor(); } + +private: + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JB) + return std::move(JB); + return llvm::make_error("TestError", std::error_code()); + } + + std::unique_ptr JB; }; -#ifdef _AIX +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { #else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { @@ -69,6 +98,8 @@ TEST(InterpreterExtensionsTest, ExecutorCreateReset) { llvm::Error ErrOut = llvm::Error::success(); TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut); cantFail(std::move(ErrOut)); + EXPECT_THAT_ERROR(Interp.testCreateJITBuilderError(), + llvm::FailedWithMessage("TestError")); cantFail(Interp.testCreateExecutor()); Interp.resetExecutor(); cantFail(Interp.testCreateExecutor()); @@ -126,4 +157,90 @@ TEST(InterpreterExtensionsTest, FindRuntimeInterface) { EXPECT_EQ(1U, Interp.RuntimeIBPtr->TransformerQueries); } +class CustomJBInterpreter : public Interpreter { + using CustomJITBuilderCreatorFunction = + std::function>()>; + CustomJITBuilderCreatorFunction JBCreator = nullptr; + +public: + CustomJBInterpreter(std::unique_ptr CI, llvm::Error &ErrOut) + : Interpreter(std::move(CI), ErrOut) {} + + ~CustomJBInterpreter() override { + // Skip cleanUp() because it would trigger LLJIT default dtors + Interpreter::ResetExecutor(); + } + + void setCustomJITBuilderCreator(CustomJITBuilderCreatorFunction Fn) { + JBCreator = std::move(Fn); + } + + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JBCreator) + return JBCreator(); + return Interpreter::CreateJITBuilder(CI); + } + + llvm::Error CreateExecutor() { return Interpreter::CreateExecutor(); } +}; + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) { +#else +TEST(InterpreterExtensionsTest, DefaultCrossJIT) { +#endif + IncrementalCompilerBuilder CB; + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + cantFail(Interp.CreateExecutor()); +} + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) { +#else +TEST(InterpreterExtensionsTest, CustomCrossJIT) { +#endif + std::string TargetTriple = "armv6-none-eabi"; + + IncrementalCompilerBuilder CB; + CB.SetTargetTriple(TargetTriple); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + + using namespace llvm::orc; + LLJIT *JIT = nullptr; + std::vector> Objs; + Interp.setCustomJITBuilderCreator([&]() { + auto JTMB = JITTargetMachineBuilder(llvm::Triple(TargetTriple)); + JTMB.setCPU("cortex-m0plus"); + auto JB = std::make_unique(); + JB->setJITTargetMachineBuilder(JTMB); + JB->setPlatformSetUp(setUpInactivePlatform); + JB->setNotifyCreatedCallback([&](LLJIT &J) { + ObjectLayer &ObjLayer = J.getObjLinkingLayer(); + auto *JITLinkObjLayer = llvm::dyn_cast(&ObjLayer); + JITLinkObjLayer->setReturnObjectBuffer( + [&Objs](std::unique_ptr MB) { + Objs.push_back(std::move(MB)); + }); + JIT = &J; + return llvm::Error::success(); + }); + return JB; + }); + + EXPECT_EQ(0U, Objs.size()); + cantFail(Interp.CreateExecutor()); + cantFail(Interp.ParseAndExecute("int a = 1;")); + ExecutorAddr Addr = cantFail(JIT->lookup("a")); + EXPECT_NE(0U, Addr.getValue()); + EXPECT_EQ(1U, Objs.size()); +} + } // end anonymous namespace -- GitLab From 75e528fdd9594ecb6fdb5d9e7bee1506f7e43be0 Mon Sep 17 00:00:00 2001 From: David Stuttard Date: Mon, 25 Mar 2024 09:01:46 +0000 Subject: [PATCH 294/933] [AMDGPU] Extend zero initialization of return values for TFE (#85759) buffer_load instructions that use TFE also need to zero initialize return values similar to how the image instructions currently work. Add support for this with standard zero init of all results + zero init of just TFE flag when enable-prt-strict-null subtarget feature is disabled. --- .../AMDGPU/AMDGPUInstructionSelector.cpp | 32 -- llvm/lib/Target/AMDGPU/BUFInstructions.td | 5 +- llvm/lib/Target/AMDGPU/MIMGInstructions.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 87 ++--- llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 6 + llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 + .../llvm.amdgcn.struct.buffer.load.format.ll | 1 + ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 1 + .../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 111 ++++-- .../llvm.amdgcn.struct.buffer.load.format.ll | 317 +++++++++++++++++- ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 280 ++++++++++++++++ 12 files changed, 741 insertions(+), 105 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 94cc1d90e0ca..e13c13913d4e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -2045,38 +2045,6 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic( if (BaseOpcode->HasD16) MIB.addImm(IsD16 ? -1 : 0); - if (IsTexFail) { - // An image load instruction with TFE/LWE only conditionally writes to its - // result registers. Initialize them to zero so that we always get well - // defined result values. - assert(VDataOut && !VDataIn); - Register Tied = MRI->cloneVirtualRegister(VDataOut); - Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) - .addImm(0); - auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); - if (STI.usePRTStrictNull()) { - // With enable-prt-strict-null enabled, initialize all result registers to - // zero. - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts) - RegSeq.addReg(Zero).addImm(Sub); - } else { - // With enable-prt-strict-null disabled, only initialize the extra TFE/LWE - // result register. - Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts.drop_back(1)) - RegSeq.addReg(Undef).addImm(Sub); - RegSeq.addReg(Zero).addImm(Parts.back()); - } - MIB.addReg(Tied, RegState::Implicit); - MIB->tieOperands(0, MIB->getNumOperands() - 1); - } - MI.eraseFromParent(); constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 4ae514ffcf78..273f92abf354 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -86,7 +86,7 @@ class BUF_Pseudo has_soffset = 1; bits<1> has_offset = 1; bits<1> has_slc = 1; - bits<1> tfe = ?; + bits<1> tfe = 0; bits<4> elements = 0; bits<1> has_sccb = 1; bits<1> sccb_value = 0; @@ -323,6 +323,7 @@ class MUBUF_Pseudo (MUBUFGetBaseOpcode.ret); let MUBUF = 1; let AsmMatchConverter = "cvtMubuf"; + let usesCustomInserter = 1; } class MUBUF_Real : @@ -3369,7 +3370,7 @@ def MUBUFInfoTable : GenericTable { let CppTypeName = "MUBUFInfo"; let Fields = [ "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", - "IsBufferInv" + "IsBufferInv", "tfe" ]; let PrimaryKey = ["Opcode"]; diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index 595ef39ce03e..23e8be0d5e45 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -210,6 +210,7 @@ class MIMG : MIMG_Base { let hasPostISelHook = 1; + let usesCustomInserter = 1; Instruction Opcode = !cast(NAME); MIMGBaseOpcode BaseOpcode; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d437f339a687..81a231f0cade 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -5410,6 +5410,11 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( return SplitBB; } default: + if (TII->isImage(MI) || TII->isMUBUF(MI)) { + if (!MI.mayStore()) + AddMemOpInit(MI); + return BB; + } return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); } } @@ -15034,60 +15039,67 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node, // result register that will be written in the case of a memory access failure. // The required code is also added to tie this init code to the result of the // img instruction. -void SITargetLowering::AddIMGInit(MachineInstr &MI) const { +void SITargetLowering::AddMemOpInit(MachineInstr &MI) const { const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo(); MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); MachineBasicBlock &MBB = *MI.getParent(); - MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); - MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); - MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - - if (!TFE && !LWE) // intersect_ray - return; + int DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + unsigned InitIdx = 0; - unsigned TFEVal = TFE ? TFE->getImm() : 0; - unsigned LWEVal = LWE ? LWE->getImm() : 0; - unsigned D16Val = D16 ? D16->getImm() : 0; + if (TII->isImage(MI)) { + MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); + MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); + MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - if (!TFEVal && !LWEVal) - return; + if (!TFE && !LWE) // intersect_ray + return; - // At least one of TFE or LWE are non-zero - // We have to insert a suitable initialization of the result value and - // tie this to the dest of the image instruction. + unsigned TFEVal = TFE ? TFE->getImm() : 0; + unsigned LWEVal = LWE ? LWE->getImm() : 0; + unsigned D16Val = D16 ? D16->getImm() : 0; - const DebugLoc &DL = MI.getDebugLoc(); + if (!TFEVal && !LWEVal) + return; - int DstIdx = - AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + // At least one of TFE or LWE are non-zero + // We have to insert a suitable initialization of the result value and + // tie this to the dest of the image instruction. - // Calculate which dword we have to initialize to 0. - MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); + // Calculate which dword we have to initialize to 0. + MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); - // check that dmask operand is found. - assert(MO_Dmask && "Expected dmask operand in instruction"); + // check that dmask operand is found. + assert(MO_Dmask && "Expected dmask operand in instruction"); - unsigned dmask = MO_Dmask->getImm(); - // Determine the number of active lanes taking into account the - // Gather4 special case - unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); + unsigned dmask = MO_Dmask->getImm(); + // Determine the number of active lanes taking into account the + // Gather4 special case + unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); - bool Packed = !Subtarget->hasUnpackedD16VMem(); + bool Packed = !Subtarget->hasUnpackedD16VMem(); - unsigned InitIdx = - D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; + InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; - // Abandon attempt if the dst size isn't large enough - // - this is in fact an error but this is picked up elsewhere and - // reported correctly. - uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; - if (DstSize < InitIdx) + // Abandon attempt if the dst size isn't large enough + // - this is in fact an error but this is picked up elsewhere and + // reported correctly. + uint32_t DstSize = + TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + if (DstSize < InitIdx) + return; + } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) { + InitIdx = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + } else { return; + } + + const DebugLoc &DL = MI.getDebugLoc(); // Create a register for the initialization value. - Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg()); unsigned NewDst = 0; // Final initialized value will be in here // If PRTStrictNull feature is enabled (the default) then initialize @@ -15185,11 +15197,8 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, return; } - if (TII->isImage(MI)) { - if (!MI.mayStore()) - AddIMGInit(MI); + if (TII->isImage(MI)) TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr); - } } static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 89da4428e3ab..9856a2923d38 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -466,7 +466,7 @@ public: SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override; - void AddIMGInit(MachineInstr &MI) const; + void AddMemOpInit(MachineInstr &MI) const; void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..a90dc32d396f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -318,6 +318,7 @@ struct MUBUFInfo { bool has_srsrc; bool has_soffset; bool IsBufferInv; + bool tfe; }; struct MTBUFInfo { @@ -466,6 +467,11 @@ bool getMUBUFIsBufferInv(unsigned Opc) { return Info ? Info->IsBufferInv : false; } +bool getMUBUFTfe(unsigned Opc) { + const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); + return Info ? Info->tfe : false; +} + bool getSMEMIsBuffer(unsigned Opc) { const SMInfo *Info = getSMEMOpcodeHelper(Opc); return Info ? Info->IsBuffer : false; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..f4f9a787100b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -525,6 +525,9 @@ bool getMUBUFHasSoffset(unsigned Opc); LLVM_READONLY bool getMUBUFIsBufferInv(unsigned Opc); +LLVM_READONLY +bool getMUBUFTfe(unsigned Opc); + LLVM_READONLY bool getSMEMIsBuffer(unsigned Opc); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll index 686b849ff58f..06bd45a45cce 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX12 %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; GFX8-LABEL: name: struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll index 9edc24554911..1e3f94a5e39c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; CHECK-LABEL: name: struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll index 1348315e72e7..7b1f55e7eeba 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll @@ -22,18 +22,36 @@ main_body: define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_both: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x00,0x00,0x60,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x05] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v9, v8 ; encoding: [0x08,0x03,0x12,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v10, v8 ; encoding: [0x08,0x03,0x14,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v11, v8 ; encoding: [0x08,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v8 ; encoding: [0x08,0x03,0x18,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:7], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x05,0x00,0x60,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x08,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_both: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x05] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0a,0x09] +; GFX12-NEXT: v_dual_mov_b32 v11, v8 :: v_dual_mov_b32 v12, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0c,0x0b] +; GFX12-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v7, v6, v5], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x07,0x06,0x05,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x08,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32i32.i32(i32 2, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0) @@ -63,18 +81,37 @@ main_body: define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, v3 ; encoding: [0x80,0x00,0x10,0xca,0x03,0x01,0x08,0x09] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v10, v9 ; encoding: [0x00,0x01,0x10,0xca,0x09,0x01,0x0a,0x05] +; GFX11-NEXT: v_mov_b32_e32 v11, v9 ; encoding: [0x09,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v9 ; encoding: [0x09,0x03,0x18,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v13, v9 ; encoding: [0x09,0x03,0x1a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:8], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x05,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x09,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2, v3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_mov_b32_e32 v9, 0 ; encoding: [0x80,0x02,0x12,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v6, v2 ; encoding: [0x03,0x01,0x10,0xca,0x02,0x01,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v0 ; encoding: [0x01,0x01,0x10,0xca,0x00,0x01,0x08,0x07] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x23,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v10, v9 :: v_dual_mov_b32 v11, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0a,0x0a] +; GFX12-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0c,0x0c] +; GFX12-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v8, v7, v6, v5], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x08,0x07,0x06,0x05] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x09,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32i32.i32(i32 8, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -155,18 +192,31 @@ main_body: define amdgpu_ps <4 x half> @load_2dmsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x03] +; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x05] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], v[3:5], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x03,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x06,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x03] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v6 ; encoding: [0x06,0x01,0x10,0xca,0x06,0x01,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x05,0x04,0x03,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x06,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -196,18 +246,31 @@ main_body: define amdgpu_ps <4 x half> @load_2darraymsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9c,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX11-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9d,0x01,0x62,0xf0,0x06,0x00,0x20,0x00,0x05,0x04,0x03,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x07,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX12-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7 ; encoding: [0x07,0x01,0x10,0xca,0x07,0x01,0x08,0x08] +; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x06,0x05,0x04,0x03] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x07,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll index 00be32b06de0..ba3d306cc0cf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-enable-prt-strict-null -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s ;RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s @@ -34,6 +35,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -75,6 +86,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -146,6 +164,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) { ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs_large: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -196,6 +233,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_12bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -235,6 +279,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_13bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -274,6 +327,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_16bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -313,6 +375,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_23bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -352,6 +423,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-SDAG-LABEL: buffer_load_voffset_large_24bit: ; GFX12-SDAG: ; %bb.0: ; %main_body ; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0x800000 :: v_dual_mov_b32 v0, 0 @@ -389,6 +469,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_idx: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], null idxen @@ -427,6 +513,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -466,6 +561,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs_imm: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -497,6 +601,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], null idxen offen @@ -529,6 +639,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i3 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both_reversed: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v2, v0 @@ -562,6 +679,13 @@ define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -595,6 +719,13 @@ define amdgpu_ps float @buffer_load_x_i32(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x_i32: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -629,6 +760,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_xy: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -644,7 +782,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 -; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX6-NEXT: v_mov_b32_e32 v7, 2 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 +; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s0, s2 @@ -658,7 +801,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 -; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX8PLUS-NEXT: v_mov_b32_e32 v7, 2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 +; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX8PLUS-NEXT: v_mov_b32_e32 v0, v6 @@ -667,22 +815,40 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 +; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v2, 2 +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4i32_tfe: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v2, 0 -; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe +; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 +; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX12-NEXT: v_mov_b32_e32 v0, v6 ; GFX12-NEXT: ; return to shader part epilog - %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0) + %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out %status = extractvalue { <4 x i32>, i32 } %load, 1 @@ -694,6 +860,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -708,6 +878,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -718,15 +892,32 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 ; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off @@ -744,6 +935,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -759,6 +953,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -769,15 +966,31 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -795,6 +1008,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -810,6 +1026,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -820,15 +1039,31 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -846,6 +1081,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -860,6 +1098,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -870,15 +1110,29 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -896,6 +1150,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -910,6 +1167,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -920,15 +1179,29 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -946,6 +1219,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -960,6 +1234,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -970,15 +1245,28 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off @@ -996,6 +1284,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -1010,6 +1299,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -1020,15 +1310,28 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll index b0bd4e428ef2..c5202b84fa1e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mattr=-enable-prt-strict-null -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) { ; GFX6-LABEL: buffer_load: @@ -31,6 +32,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp ; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0) %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1) @@ -62,6 +73,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 42, i32 0, i32 0) ret <4 x float> %data @@ -126,6 +144,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg) ; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog main_body: %d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0) %d.1 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 32764, i32 0) @@ -156,6 +193,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 0, i32 0) ret <4 x float> %data @@ -188,6 +232,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8188, i32 0, i32 0) ret <4 x float> %data @@ -220,6 +273,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 65532, i32 0, i32 0) ret <4 x float> %data @@ -252,6 +314,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8388604, i32 0, i32 0) ret <4 x float> %data @@ -284,6 +355,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 16777212, i32 0, i32 0) ret <4 x float> %data @@ -307,6 +387,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0, i32 0) ret <4 x float> %data @@ -339,6 +425,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -371,6 +466,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %ofs = add i32 %1, 60 %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %ofs, i32 0, i32 0) @@ -395,6 +499,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(ptr addrspace(8) inreg, i32, i32) ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 %2, i32 0, i32 0) ret <4 x float> %data @@ -421,6 +531,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(ptr addrspace(8) inreg, ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %2, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -447,6 +564,13 @@ define amdgpu_ps float @buffer_load_x(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret float %data @@ -473,6 +597,13 @@ define amdgpu_ps float @buffer_load_x_i32(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call i32 @llvm.amdgcn.struct.ptr.buffer.load.format.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %fdata = bitcast i32 %data to float @@ -500,6 +631,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret <2 x float> %data @@ -509,6 +647,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -523,6 +665,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -533,11 +679,25 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out @@ -550,6 +710,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -564,6 +728,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -574,11 +742,25 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x float>, i32 } %load, 0 store <4 x float> %data, ptr addrspace(1) %out @@ -591,6 +773,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -606,6 +791,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -616,11 +804,24 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x i32>, i32 } %load, 0 store <3 x i32> %data, ptr addrspace(1) %out @@ -633,6 +834,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -648,6 +852,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -658,11 +865,24 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x float>, i32 } %load, 0 store <3 x float> %data, ptr addrspace(1) %out @@ -675,6 +895,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -689,6 +912,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -699,11 +924,23 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x i32>, i32 } %load, 0 store <2 x i32> %data, ptr addrspace(1) %out @@ -716,6 +953,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -730,6 +970,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -740,11 +982,23 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x float>, i32 } %load, 0 store <2 x float> %data, ptr addrspace(1) %out @@ -757,6 +1011,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -771,6 +1026,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -781,11 +1037,22 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { i32, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { i32, i32 } %load, 0 store i32 %data, ptr addrspace(1) %out @@ -798,6 +1065,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -812,6 +1080,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -822,11 +1091,22 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { float, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { float, i32 } %load, 0 store float %data, ptr addrspace(1) %out -- GitLab From 99c40f6ba60ea6e5dbd3f015956c7d2f6e25e54c Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:13:42 +0100 Subject: [PATCH 295/933] [SPIR-V] Introduce a command line option to support compatibility with Khronos SPIRV Translator (#86101) SPIRV-LLVM-Translator project (https://github.com/KhronosGroup/SPIRV-LLVM-Translator) from Khronos Group is a tool and a library for bi-directional translation between SPIR-V and LLVM IR. In its backward translation from SPIR-V to LLVM IR SPIRV-LLVM-Translator isn't necessarily able to cover the same SPIR-V patterns/instructions set that SPIRV Backend produces, even if we target the same SPIR-V version in both SPIRV-LLVM-Translator and SPIRV Backend projects. To improve interoperability and ability to apply SPIRV Backend output in different products this PR introduces a notion of a mode of SPIR-V output that is compatible with a subset of SPIR-V supported by SPIRV-LLVM-Translator. This includes a new command line option that doesn't influence default behavior of SPIRV Backend and one test case that demonstrates how this command line option may be used to get a practical benefit of producing that one of two possible and similar output options that can be understood by SPIRV-LLVM-Translator. --- llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp | 8 +++++++- llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp index 38caa7c8ea0a..09a029a35a74 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -27,6 +27,11 @@ using namespace llvm; #define GET_SUBTARGETINFO_CTOR #include "SPIRVGenSubtargetInfo.inc" +static cl::opt + SPVTranslatorCompat("translator-compatibility-mode", + cl::desc("SPIR-V Translator compatibility mode"), + cl::Optional, cl::init(false)); + cl::list Extensions( "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore, cl::Hidden, @@ -157,8 +162,9 @@ bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const { } // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual. +// In SPIR-V Translator compatibility mode this feature is not available. bool SPIRVSubtarget::canDirectlyComparePointers() const { - return isAtLeastVer(SPIRVVersion, 14); + return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, 14); } void SPIRVSubtarget::initAvailableExtensions() { diff --git a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll index 641e2bf0649c..31cd8bd45929 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll @@ -1,7 +1,13 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - | FileCheck %s --check-prefix=CHECK-COMPAT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[EQ:%.*]] "test_eq" ; CHECK-DAG: OpName [[NE:%.*]] "test_ne" +; CHECK-COMPAT-DAG: OpName [[EQ:%.*]] "test_eq" +; CHECK-COMPAT-DAG: OpName [[NE:%.*]] "test_ne" ; CHECK-DAG: OpName [[ULT:%.*]] "test_ult" ; CHECK-DAG: OpName [[SLT:%.*]] "test_slt" ; CHECK-DAG: OpName [[ULE:%.*]] "test_ule" @@ -19,6 +25,9 @@ ; CHECK-NEXT: [[R:%.*]] = OpPtrEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[EQ]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_eq(i16* %a, i16* %b) { %r = icmp eq i16* %a, %b ret i1 %r @@ -31,6 +40,9 @@ define i1 @test_eq(i16* %a, i16* %b) { ; CHECK-NEXT: [[R:%.*]] = OpPtrNotEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[NE]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrNotEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_ne(i16* %a, i16* %b) { %r = icmp ne i16* %a, %b ret i1 %r -- GitLab From 1d250d9099a9ba8b53add7eb7db6827e8fc0c8fd Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:08 +0100 Subject: [PATCH 296/933] [SPIR-V] Improve type inference in SPIR-V Backend for opaque pointers (#86283) This PR improves type inference in SPIR-V Backend for opaque pointers, accounting or a case when there is a chain of function calls that allows to deduce formal parameter types from actual arguments. The attached test demonstrates the case. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 123 +++++++++++------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 12 +- .../pointers/type-deduce-by-call-chain.ll | 52 ++++++++ 4 files changed, 138 insertions(+), 50 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 458af9229ed7..5828db6669ff 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -92,6 +92,9 @@ class SPIRVEmitIntrinsics void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); void processParamTypes(Function *F, IRBuilder<> &B); + Type *deduceFunParamType(Function *F, unsigned OpIdx); + Type *deduceFunParamType(Function *F, unsigned OpIdx, + std::unordered_set &FVisited); public: static char ID; @@ -169,6 +172,10 @@ static inline void reportFatalOnTokenType(const Instruction *I) { static Type *deduceElementTypeHelper(Value *I, std::unordered_set &Visited, DenseMap &DeducedElTys) { + // allow to pass nullptr as an argument + if (!I) + return nullptr; + // maybe already known auto It = DeducedElTys.find(I); if (It != DeducedElTys.end()) @@ -182,15 +189,20 @@ static Type *deduceElementTypeHelper(Value *I, // fallback value in case when we fail to deduce a type Type *Ty = nullptr; // look for known basic patterns of type inference - if (auto *Ref = dyn_cast(I)) + if (auto *Ref = dyn_cast(I)) { Ty = Ref->getAllocatedType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getResultElementType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getValueType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, DeducedElTys); + } else if (auto *Ref = dyn_cast(I)) { + if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); + isPointerTy(Src) && isPointerTy(Dest)) + Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, DeducedElTys); + } // remember the found relationship if (Ty) @@ -795,61 +807,80 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, } } -void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { - DenseMap Args; - unsigned i = 0; - for (Argument &Arg : F->args()) { - if (isUntypedPointerTy(Arg.getType()) && - DeducedElTys.find(&Arg) == DeducedElTys.end() && - !HasPointeeTypeAttr(&Arg)) - Args[i] = &Arg; - i++; - } - if (Args.size() == 0) - return; +Type *SPIRVEmitIntrinsics::deduceFunParamType(Function *F, unsigned OpIdx) { + std::unordered_set FVisited; + return deduceFunParamType(F, OpIdx, FVisited); +} + +Type *SPIRVEmitIntrinsics::deduceFunParamType( + Function *F, unsigned OpIdx, std::unordered_set &FVisited) { + // maybe a cycle + if (FVisited.find(F) != FVisited.end()) + return nullptr; + FVisited.insert(F); - // Args contains opaque pointers without element type definition - B.SetInsertPointPastAllocas(F); std::unordered_set Visited; + SmallVector> Lookup; + // search in function's call sites for (User *U : F->users()) { CallInst *CI = dyn_cast(U); - if (!CI) + if (!CI || OpIdx >= CI->arg_size()) continue; - for (unsigned OpIdx = 0; OpIdx < CI->arg_size() && Args.size() > 0; - OpIdx++) { - auto It = Args.find(OpIdx); - Argument *Arg = It == Args.end() ? nullptr : It->second; - if (!Arg) - continue; - Value *OpArg = CI->getArgOperand(OpIdx); - if (!isPointerTy(OpArg->getType())) + Value *OpArg = CI->getArgOperand(OpIdx); + if (!isPointerTy(OpArg->getType())) + continue; + // maybe we already know operand's element type + if (auto It = DeducedElTys.find(OpArg); It != DeducedElTys.end()) + return It->second; + // search in actual parameter's users + for (User *OpU : OpArg->users()) { + Instruction *Inst = dyn_cast(OpU); + if (!Inst || Inst == CI) continue; - // maybe we already know the operand's element type - auto DeducedIt = DeducedElTys.find(OpArg); - Type *ElemTy = - DeducedIt == DeducedElTys.end() ? nullptr : DeducedIt->second; - if (!ElemTy) { - for (User *OpU : OpArg->users()) { - if (Instruction *Inst = dyn_cast(OpU)) { - Visited.clear(); - ElemTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys); - if (ElemTy) - break; - } - } + Visited.clear(); + if (Type *Ty = deduceElementTypeHelper(Inst, Visited, DeducedElTys)) + return Ty; + } + // check if it's a formal parameter of the outer function + if (!CI->getParent() || !CI->getParent()->getParent()) + continue; + Function *OuterF = CI->getParent()->getParent(); + if (FVisited.find(OuterF) != FVisited.end()) + continue; + for (unsigned i = 0; i < OuterF->arg_size(); ++i) { + if (OuterF->getArg(i) == OpArg) { + Lookup.push_back(std::make_pair(OuterF, i)); + break; } - if (ElemTy) { - unsigned AddressSpace = getPointerAddressSpace(Arg->getType()); + } + } + + // search in function parameters + for (auto &Pair : Lookup) { + if (Type *Ty = deduceFunParamType(Pair.first, Pair.second, FVisited)) + return Ty; + } + + return nullptr; +} + +void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { + B.SetInsertPointPastAllocas(F); + DenseMap Args; + for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { + Argument *Arg = F->getArg(OpIdx); + if (isUntypedPointerTy(Arg->getType()) && + DeducedElTys.find(Arg) == DeducedElTys.end() && + !HasPointeeTypeAttr(Arg)) { + if (Type *ElemTy = deduceFunParamType(F, OpIdx)) { CallInst *AssignPtrTyCI = buildIntrWithMD( Intrinsic::spv_assign_ptr_type, {Arg->getType()}, - Constant::getNullValue(ElemTy), Arg, {B.getInt32(AddressSpace)}, B); + Constant::getNullValue(ElemTy), Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); DeducedElTys[AssignPtrTyCI] = ElemTy; DeducedElTys[Arg] = ElemTy; - Args.erase(It); } } - if (Args.size() == 0) - break; } } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 42f8397a3023..ee52163a5d12 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -479,6 +479,7 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( GVar = M->getGlobalVariable(Name); if (GVar == nullptr) { const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type. + // Module takes ownership of the global var. GVar = new GlobalVariable(*M, const_cast(Ty), false, GlobalValue::ExternalLinkage, nullptr, Twine(Name)); diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 5bb8f6084f96..39228e2196b3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -499,6 +499,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); Register GV = I.getOperand(1).getReg(); MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); + (void)II; assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || (*II).getOpcode() == TargetOpcode::COPY || (*II).getOpcode() == SPIRV::OpVariable) && @@ -771,10 +772,13 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); // TODO: check if we have such GV, add init, use buildGlobalVariable. - Type *LLVMArrTy = ArrayType::get( - IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); - GlobalVariable *GV = - new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); + Function &CurFunction = GR.CurMF->getFunction(); + Type *LLVMArrTy = + ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); + // Module takes ownership of the global var. + GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, + true, GlobalValue::InternalLinkage, + Constant::getNullValue(LLVMArrTy)); Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(GV, GR.CurMF, VarReg); diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll new file mode 100644 index 000000000000..703f1e22a032 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll @@ -0,0 +1,52 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[ArgCum:.*]] "_arg_cum" +; CHECK-SPIRV-DAG: OpName %[[FunTest:.*]] "test" +; CHECK-SPIRV-DAG: OpName %[[Addr:.*]] "addr" +; CHECK-SPIRV-DAG: OpName %[[StubObj:.*]] "stub_object" +; CHECK-SPIRV-DAG: OpName %[[MemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooStub:.*]] "foo_stub" +; CHECK-SPIRV-DAG: OpName %[[FooObj:.*]] "foo_object" +; CHECK-SPIRV-DAG: OpName %[[FooMemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooFunc:.*]] "foo" +; CHECK-SPIRV-DAG: %[[TyLong:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[TyVoid:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[TyPtrLong:.*]] = OpTypePointer CrossWorkgroup %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunPtrLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrLong]] +; CHECK-SPIRV-DAG: %[[TyGenPtrLong:.*]] = OpTypePointer Generic %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunGenPtrLongLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyGenPtrLong]] %[[TyLong]] +; CHECK-SPIRV-DAG: %[[Const3:.*]] = OpConstant %[[TyLong]] 3 +; CHECK-SPIRV: %[[FunTest]] = OpFunction %[[TyVoid]] None %[[TyFunPtrLong]] +; CHECK-SPIRV: %[[ArgCum]] = OpFunctionParameter %[[TyPtrLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooFunc]] %[[Addr]] %[[Const3]] +; CHECK-SPIRV: %[[FooStub]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[StubObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[MemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: %[[FooFunc]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[FooObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[FooMemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooStub]] %[[FooObj]] %[[FooMemOrder]] + +define spir_kernel void @test(ptr addrspace(1) noundef align 4 %_arg_cum) { +entry: + %lptr = getelementptr inbounds i32, ptr addrspace(1) %_arg_cum, i64 1 + %addr = addrspacecast ptr addrspace(1) %lptr to ptr addrspace(4) + %object = bitcast ptr addrspace(4) %addr to ptr addrspace(4) + call spir_func void @foo(ptr addrspace(4) %object, i32 3) + ret void +} + +define void @foo_stub(ptr addrspace(4) noundef %stub_object, i32 noundef %mem_order) { +entry: + %object.addr = alloca ptr addrspace(4) + %object.addr.ascast = addrspacecast ptr %object.addr to ptr addrspace(4) + store ptr addrspace(4) %stub_object, ptr addrspace(4) %object.addr.ascast + ret void +} + +define void @foo(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) { + tail call void @foo_stub(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) + ret void +} + -- GitLab From b0d03ccc0855f2bff39160f25fcde06aae07cace Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:46 +0100 Subject: [PATCH 297/933] [SPIR-V] Fix illegal OpConstantComposite instruction with non-const constituents in SPIR-V Backend (#86352) This PR fixes illegal use of OpConstantComposite with non-constant constituents. The test attached to the PR is able now to satisfy `spirv-val` check. Before the fix SPIR-V Backend produced for the attached test case a pattern like ``` %a = OpVariable %_ptr_CrossWorkgroup_uint CrossWorkgroup %uint_123 %11 = OpConstantComposite %_struct_6 %a %a ``` so that `spirv-val` complained with ``` error: line 25: OpConstantComposite Constituent '10[%a]' is not a constant or undef. %11 = OpConstantComposite %_struct_6 %a %a ``` --- .../Target/SPIRV/SPIRVDuplicatesTracker.cpp | 1 + .../lib/Target/SPIRV/SPIRVDuplicatesTracker.h | 9 ++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 8 ++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 90 +++++++++++++++---- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 1 + .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 1 + .../SPIRV/pointers/struct-opaque-pointers.ll | 2 +- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp index d82fb2df4539..7c32bb1968ef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp @@ -39,6 +39,7 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph( prebuildReg2Entry(GT, Reg2Entry); prebuildReg2Entry(FT, Reg2Entry); prebuildReg2Entry(AT, Reg2Entry); + prebuildReg2Entry(MT, Reg2Entry); prebuildReg2Entry(ST, Reg2Entry); for (auto &Op2E : Reg2Entry) { diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h index 96cc621791e9..2ec3fb35ca04 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h @@ -262,6 +262,7 @@ class SPIRVGeneralDuplicatesTracker { SPIRVDuplicatesTracker GT; SPIRVDuplicatesTracker FT; SPIRVDuplicatesTracker AT; + SPIRVDuplicatesTracker MT; SPIRVDuplicatesTracker ST; // NOTE: using MOs instead of regs to get rid of MF dependency to be able @@ -306,6 +307,10 @@ public: AT.add(Arg, MF, R); } + void add(const MachineInstr *MI, const MachineFunction *MF, Register R) { + MT.add(MI, MF, R); + } + void add(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF, Register R) { ST.add(TD, MF, R); @@ -337,6 +342,10 @@ public: return AT.find(const_cast(Arg), MF); } + Register find(const MachineInstr *MI, const MachineFunction *MF) { + return MT.find(const_cast(MI), MF); + } + Register find(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF) { return ST.find(TD, MF); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index ee52163a5d12..db66ed4f0e0f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -123,6 +123,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVType *ElemType, MachineIRBuilder &MIRBuilder) { auto EleOpc = ElemType->getOpcode(); + (void)EleOpc; assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat || EleOpc == SPIRV::OpTypeBool) && "Invalid vector element type"); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index da480b22a525..ed0f90ff89ce 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -94,6 +94,14 @@ public: DT.add(Arg, MF, R); } + void add(const MachineInstr *MI, MachineFunction *MF, Register R) { + DT.add(MI, MF, R); + } + + Register find(const MachineInstr *MI, MachineFunction *MF) { + return DT.find(MI, MF); + } + Register find(const Constant *C, MachineFunction *MF) { return DT.find(C, MF); } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 39228e2196b3..505b19a4d66e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -231,6 +231,9 @@ private: Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const; + + bool wrapIntoSpecConstantOp(MachineInstr &I, + SmallVector &CompositeArgs) const; }; } // end anonymous namespace @@ -1249,6 +1252,24 @@ static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, return N; } +// Return true if the type represents a constant register +static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || + OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; +} + +// Return true if the virtual register represents a constant +static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { + if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) + return isConstReg(MRI, OpDef); + return false; +} + bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1266,16 +1287,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, // check if we may construct a constant vector Register OpReg = I.getOperand(OpIdx).getReg(); - bool IsConst = false; - if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) { - if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && - OpDef->getOperand(1).isReg()) { - if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) - OpDef = RefDef; - } - IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT || - OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; - } + bool IsConst = isConstReg(MRI, OpReg); if (!IsConst && N < 2) report_fatal_error( @@ -1628,6 +1640,48 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg, return Res.constrainAllUses(TII, TRI, RBI); } +// Maybe wrap a value into OpSpecConstantOp +bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( + MachineInstr &I, SmallVector &CompositeArgs) const { + bool Result = true; + unsigned Lim = I.getNumExplicitOperands(); + for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { + Register OpReg = I.getOperand(i).getReg(); + SPIRVType *OpDefine = MRI->getVRegDef(OpReg); + SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); + if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || + OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { + // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed + // by selectAddrSpaceCast() + CompositeArgs.push_back(OpReg); + continue; + } + MachineFunction *MF = I.getMF(); + Register WrapReg = GR.find(OpDefine, MF); + if (WrapReg.isValid()) { + CompositeArgs.push_back(WrapReg); + continue; + } + // Create a new register for the wrapper + WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); + GR.add(OpDefine, MF, WrapReg); + CompositeArgs.push_back(WrapReg); + // Decorate the wrapper register and generate a new instruction + MRI->setType(WrapReg, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); + MachineBasicBlock &BB = *I.getParent(); + Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) + .addDef(WrapReg) + .addUse(GR.getSPIRVTypeID(OpType)) + .addImm(static_cast(SPIRV::Opcode::Bitcast)) + .addUse(OpReg) + .constrainAllUses(TII, TRI, RBI); + if (!Result) + break; + } + return Result; +} + bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1666,17 +1720,21 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_const_composite: { // If no values are attached, the composite is null constant. bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); - unsigned Opcode = - IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; + // Select a proper instruction. + unsigned Opcode = SPIRV::OpConstantNull; + SmallVector CompositeArgs; + if (!IsNull) { + Opcode = SPIRV::OpConstantComposite; + if (!wrapIntoSpecConstantOp(I, CompositeArgs)) + return false; + } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // skip type MD node we already used when generated assign.type for this if (!IsNull) { - for (unsigned i = I.getNumExplicitDefs() + 1; - i < I.getNumExplicitOperands(); ++i) { - MIB.addUse(I.getOperand(i).getReg()); - } + for (Register OpReg : CompositeArgs) + MIB.addUse(OpReg); } return MIB.constrainAllUses(TII, TRI, RBI); } diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index d547f91ba4a5..1f0d8d8cd43a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -543,6 +543,7 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR, Register Dst = ICMP->getOperand(0).getReg(); MachineOperand &PredOp = ICMP->getOperand(1); const auto CC = static_cast(PredOp.getPredicate()); + (void)CC; assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) && MRI.hasOneUse(Dst) && MRI.hasOneDef(CompareReg)); uint64_t Value = getIConstVal(ICMP->getOperand(3).getReg(), &MRI); diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 8dbbd9049844..ff102e318469 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1611,3 +1611,4 @@ multiclass OpcodeOperand value> { // TODO: implement other mnemonics. defm InBoundsPtrAccessChain : OpcodeOperand<70>; defm PtrCastToGeneric : OpcodeOperand<121>; +defm Bitcast : OpcodeOperand<124>; diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll index d426fc4dfd4e..ce3ab8895a59 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]] -- GitLab From cbbbab349e9c412729c3969008cdcb677cc55790 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 12:41:32 +0000 Subject: [PATCH 298/933] [RemoveDIs] Enable direct-to-bitcode writing by default Follow on from #83251. This patch simply enables the behaviour by default in order to provide an easily revertible capstone. --- llvm/lib/IR/BasicBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index f088c7a2cc4e..ae99267f5ba8 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -39,7 +39,7 @@ cl::opt bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, - cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true)); DbgMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && -- GitLab From 3cb024198f6a028089779c0a3cb4d8e753e87c73 Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Mon, 25 Mar 2024 17:17:36 +0800 Subject: [PATCH 299/933] [RISCV][CostModel] Estimate cost of llvm.vector.reduce.fmaximum/fminimum (#80697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ‘llvm.vector.reduce.fmaximum/fminimum.*’ intrinsics propagate NaNs if any element of the vector is a NaN. Following #79402, the patch adds the cost for NaN check (vmfne + vcpop) --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 +++++++++ .../CostModel/RISCV/reduce-fmaximum.ll | 91 +++++++++++++------ .../CostModel/RISCV/reduce-fminimum.ll | 52 +++++------ 3 files changed, 136 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 8f46fdc2f7ca..f75b3d3caa62 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1001,6 +1001,51 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind); } + if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) { + SmallVector Opcodes; + InstructionCost ExtraCost = 0; + switch (IID) { + case Intrinsic::maximum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DstTy->getScalarSizeInBits(); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + + case Intrinsic::minimum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + } + return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind); + } + // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll index 1618c3833a97..f91f13b2d9ec 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll @@ -6,23 +6,37 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) @@ -32,6 +46,13 @@ define float @reduce_fmaximum_f32(float %arg) { %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ret float undef } declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>) @@ -44,21 +65,33 @@ declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) @@ -67,6 +100,12 @@ define double @reduce_fmaximum_f64(double %arg) { %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ret double undef } declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll index 35b18645b1f2..86b84025ad54 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll @@ -6,23 +6,23 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) @@ -44,21 +44,21 @@ declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -- GitLab From e550022b68fc75d32e44faa59ed8f02036cd8f26 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:26:27 +0000 Subject: [PATCH 300/933] [RemoveDIs] Load into new debug info format by default in llvm-dis (#86276) Directly load all bitcode into the new debug info format in llvm-dis. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-dis/llvm-dis.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp index 8e443318dd7d..49154dc46c57 100644 --- a/llvm/tools/llvm-dis/llvm-dis.cpp +++ b/llvm/tools/llvm-dis/llvm-dis.cpp @@ -82,6 +82,8 @@ static cl::opt PrintThinLTOIndexOnly( extern cl::opt WriteNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + namespace { static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) { @@ -169,6 +171,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&DisCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + LLVMContext Context; Context.setDiagnosticHandler( std::make_unique(argv[0])); -- GitLab From 2ef612050844355906e4b67d892a00bbb58c41d6 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:28:01 +0000 Subject: [PATCH 301/933] [RemoveDIs] Do not load into new debug info format from bitcode by default (#86268) This is NFC right now, as the global default behaviour is also "do not load into the new debug info format by default", but we want to change that soon. Additionally unconditionally convert from the new debug info format into if we've loaded into it (e.g., if the bitcode file loaded was already in the new format). The latter change is needed because verify-uselistorder doesn't yet understand DbgRecords (it doesn't know how to map them). The former change is needed because if we load from an old debug format bitcode file but load directly into the new format _and then convert back to the old mode after_, the use-lists of the debug intrinsic functions (the functions' global value uses) change. --- .../verify-uselistorder/verify-uselistorder.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index d929ae09958a..8b299c394e4e 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,6 +68,8 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); +extern cl::opt LoadBitcodeIntoNewDbgInforFormat; + namespace { struct TempFile { @@ -169,8 +171,7 @@ std::unique_ptr TempFile::readBitcode(LLVMContext &Context) const { // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!ModuleOr.get()->IsNewDbgInfoFormat && - "Unexpectedly in new debug info mode"); + ModuleOr.get()->setIsNewDbgInfoFormat(false); return std::move(ModuleOr.get()); } @@ -182,7 +183,7 @@ std::unique_ptr TempFile::readAssembly(LLVMContext &Context) const { Err.print("verify-uselistorder", errs()); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); return M; } @@ -544,6 +545,10 @@ int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "llvm tool to verify use-list order\n"); + // Do not load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + LLVMContext Context; SMDiagnostic Err; @@ -551,7 +556,7 @@ int main(int argc, char **argv) { std::unique_ptr M = parseIRFile(InputFilename, Err, Context); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); if (!M.get()) { Err.print(argv[0], errs()); -- GitLab From 8263a883342d9925a4a1fd9752efc8deda5840fc Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:29:19 +0000 Subject: [PATCH 302/933] [RemoveDIs] Load into new debug info format by default in llvm-link (#86274) Directly load all bitcode into the new debug info format in llvm-link. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-link/llvm-link.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp index 9e7f2c3ebac4..9049cb5e8580 100644 --- a/llvm/tools/llvm-link/llvm-link.cpp +++ b/llvm/tools/llvm-link/llvm-link.cpp @@ -136,6 +136,8 @@ static cl::opt TryUseNewDbgInfoFormat( extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + static ExitOnError ExitOnErr; // Read the specified bitcode file in and return it. This routine searches the @@ -480,6 +482,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&LinkCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. if (TryUseNewDbgInfoFormat) { -- GitLab From 772e316457ef94759804d9f4da0af70d8d2ca4d4 Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Mon, 25 Mar 2024 09:43:41 +0000 Subject: [PATCH 303/933] [FMV] Allow multi versioning without default declaration. (#85454) This was a limitation which has now been lifted. Please read the thread below for more details: https://github.com/llvm/llvm-project/pull/84405#discussion_r1525583647 Basically it allows to separate versioned implementations across different TUs without having to share private header files which contain the default declaration. The ACLE spec has been updated accordingly to make this explicit: "Each version declaration should be visible at the translation unit in which the corresponding function version resides." https://github.com/ARM-software/acle/pull/310 If a resolver is required (because there is a caller in the TU), then a default declaration is implicitly generated. --- clang/lib/CodeGen/CodeGenModule.cpp | 131 ++++--- clang/lib/Sema/SemaDecl.cpp | 6 +- clang/lib/Sema/SemaOverload.cpp | 44 ++- clang/test/CodeGen/attr-target-version.c | 368 ++++++++++++------ clang/test/CodeGenCXX/attr-target-version.cpp | 62 +-- clang/test/Sema/aarch64-sme-func-attrs.c | 8 +- clang/test/Sema/attr-target-version.c | 8 +- clang/test/SemaCXX/attr-target-version.cpp | 11 +- 8 files changed, 403 insertions(+), 235 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index cb153066b28d..ac81df8cf7ad 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3711,7 +3711,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + !FD->isTargetVersionMultiVersion()) return; StringRef MangledName = getMangledName(GD); @@ -4092,6 +4093,23 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM, return llvm::GlobalValue::WeakODRLinkage; } +static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) { + DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl(); + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + StorageClass SC = FD->getStorageClass(); + DeclarationName Name = FD->getNameInfo().getName(); + + FunctionDecl *NewDecl = + FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(), + FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC); + + NewDecl->setIsMultiVersion(); + NewDecl->addAttr(TargetVersionAttr::CreateImplicit( + NewDecl->getASTContext(), "default", NewDecl->getSourceRange())); + + return NewDecl; +} + void CodeGenModule::emitMultiVersionFunctions() { std::vector MVFuncsToEmit; MultiVersionFuncs.swap(MVFuncsToEmit); @@ -4099,70 +4117,54 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - bool EmitResolver = !FD->isTargetVersionMultiVersion(); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + return cast(Func); + }; + + bool HasDefaultDecl = !FD->isTargetVersionMultiVersion(); + bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion(); SmallVector Options; if (FD->isTargetMultiVersion()) { getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr(); - llvm::SmallVector Feats; + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector Feats; + llvm::Function *Func = createFunction(CurFD); + + if (const auto *TA = CurFD->getAttr()) { TA->getAddedFeatures(Feats); - Options.emplace_back(cast(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr(); - if (CurFD->isUsed() || (TVA->isDefaultVersion() && - CurFD->doesThisDeclarationHaveABody())) - EmitResolver = true; - llvm::SmallVector Feats; + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr()) { + bool HasDefaultDef = TVA->isDefaultVersion() && + CurFD->doesThisDeclarationHaveABody(); + HasDefaultDecl |= TVA->isDefaultVersion(); + ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef); TVA->getFeatures(Feats); - Options.emplace_back(cast(Func), - /*Architecture*/ "", Feats); - } + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else + llvm_unreachable("unexpected MultiVersionKind"); }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) + } else if (const auto *TC = FD->getAttr()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } + llvm::Function *Func = createFunction(FD, I); + StringRef Version = TC->getFeatureStr(I); StringRef Architecture; llvm::SmallVector Feature; @@ -4180,16 +4182,23 @@ void CodeGenModule::emitMultiVersionFunctions() { Feature.push_back(Version); } - Options.emplace_back(cast(Func), Architecture, Feature); + Options.emplace_back(Func, Architecture, Feature); } } else { assert(0 && "Expected a target or target_clones multiversion function"); continue; } - if (!EmitResolver) + if (!ShouldEmitResolver) continue; + if (!HasDefaultDecl) { + FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD); + llvm::Function *Func = createFunction(NewFD); + llvm::SmallVector Feats; + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); @@ -4480,7 +4489,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) + if (FD->isTargetVersionMultiVersion() && !FD->isUsed()) + AddDeferredMultiVersionResolverToEmit(GD); + else if (!IsForDefinition) return GetOrCreateMultiVersionResolver(GD); } } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index aa754d47a0c4..73ea155053d7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -11441,9 +11441,9 @@ static bool CheckMultiVersionFirstFunction(Sema &S, FunctionDecl *FD) { "Function lacks multiversion attribute"); const auto *TA = FD->getAttr(); const auto *TVA = FD->getAttr(); - // Target and target_version only causes MV if it is default, otherwise this - // is a normal function. - if ((TA && !TA->isDefaultVersion()) || (TVA && !TVA->isDefaultVersion())) + // The target attribute only causes MV if this declaration is the default, + // otherwise it is treated as a normal function. + if (TA && !TA->isDefaultVersion()) return false; if ((TA || TVA) && CheckMultiVersionValue(S, FD)) { diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index f6bd85bdc646..51450e486eae 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -6865,6 +6865,32 @@ static bool IsAcceptableNonMemberOperatorCandidate(ASTContext &Context, return false; } +static bool isNonViableMultiVersionOverload(FunctionDecl *FD) { + if (FD->isTargetMultiVersionDefault()) + return false; + + if (!FD->getASTContext().getTargetInfo().getTriple().isAArch64()) + return FD->isTargetMultiVersion(); + + if (!FD->isMultiVersion()) + return false; + + // Among multiple target versions consider either the default, + // or the first non-default in the absence of default version. + unsigned SeenAt = 0; + unsigned I = 0; + bool HasDefault = false; + FD->getASTContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + if (FD == CurFD) + SeenAt = I; + else if (CurFD->isTargetMultiVersionDefault()) + HasDefault = true; + ++I; + }); + return HasDefault || SeenAt != 0; +} + /// AddOverloadCandidate - Adds the given function to the set of /// candidate functions, using the given function call arguments. If /// @p SuppressUserConversions, then don't allow user-defined @@ -6970,11 +6996,7 @@ void Sema::AddOverloadCandidate( } } - if (Function->isMultiVersion() && - ((Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()) || - (Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Function)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; return; @@ -7637,11 +7659,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, return; } - if (Method->isMultiVersion() && - ((Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()) || - (Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Method)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } @@ -8127,11 +8145,7 @@ void Sema::AddConversionCandidate( return; } - if (Conversion->isMultiVersion() && - ((Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()) || - (Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Conversion)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index 25129605e76c..dd4cbbf5a898 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -109,21 +109,47 @@ int unused_with_implicit_default_def(void) { return 1; } int unused_with_implicit_forward_default_def(void) { return 0; } __attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; } -// This should generate a normal function. +// This should generate a target version despite the default not being declared. __attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; } +// These shouldn't generate anything. +int unused_version_declarations(void); +__attribute__((target_version("jscvt"))) int unused_version_declarations(void); +__attribute__((target_version("rdma"))) int unused_version_declarations(void); + +// These should generate the default (mangled) version and the resolver. +int default_def_with_version_decls(void) { return 0; } +__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void); +__attribute__((target_version("rdma"))) int default_def_with_version_decls(void); + +// The following is guarded because in NOFMV we get errors for calling undeclared functions. +#ifdef __HAVE_FUNCTION_MULTI_VERSIONING +// This should generate a default declaration, two target versions and the resolver. +__attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; } +__attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; } + +// This should generate a default declaration and the resolver. +__attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void); +__attribute__((target_version("rdma"))) int used_decl_without_default_decl(void); + +int caller(void) { return used_def_without_default_decl() + used_decl_without_default_decl(); } +#endif + //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @fmv.ifunc = weak_odr alias i32 (), ptr @fmv // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e +// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline -// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def // CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def // CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def +// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls +// CHECK: @used_def_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_def_without_default_decl +// CHECK: @used_decl_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_decl_without_default_decl // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver @@ -131,97 +157,121 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver +// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver +// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver +// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver //. // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 -// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-SAME: () #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata -// CHECK-SAME: () #[[ATTR2:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag -// CHECK-SAME: () #[[ATTR3:[0-9]+]] { +// CHECK-SAME: () #[[ATTR4:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp -// CHECK-SAME: () #[[ATTR4:[0-9]+]] { +// CHECK-SAME: () #[[ATTR5:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v -// CHECK-SAME: () #[[ATTR5:[0-9]+]] { +// CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti -// CHECK-SAME: () #[[ATTR6:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 -// CHECK-SAME: () #[[ATTR7:[0-9]+]] { +// CHECK-SAME: () #[[ATTR8:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb -// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-SAME: () #[[ATTR10:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh -// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd -// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -371,35 +421,49 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d.default +// CHECK-SAME: () #[[ATTR11]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -587,7 +651,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -595,7 +659,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -606,7 +670,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -623,228 +687,268 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR12:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 +// CHECK-SAME: () #[[ATTR12]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm +// CHECK-SAME: () #[[ATTR18:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 -// CHECK-SAME: () #[[ATTR10]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm +// CHECK-SAME: () #[[ATTR18]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_without_default -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@caller +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() +// CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() +// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] +// CHECK-NEXT: ret i32 [[ADD]] +// +// +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_def_without_default_decl.default +// +// +// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl.default // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR19:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR20:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR13]] { +// CHECK-SAME: () #[[ATTR15]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd -// CHECK-SAME: () #[[ATTR3]] { +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 14 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR34:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-SAME: () #[[ATTR35:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR9]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 -// -// // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -887,6 +991,28 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def.default // // +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @default_def_with_version_decls.default +// +// // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { @@ -995,40 +1121,50 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NOFMV-NEXT: entry: // CHECK-NOFMV-NEXT: ret i32 0 // +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 0 +// //. -// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index e06121d1a719..8b7273fe3bb5 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -35,7 +35,7 @@ struct MyClass { int unused_with_implicit_forward_default_def(void); int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void); - // This should generate a normal function. + // This should generate a target version despite the default not being declared. int __attribute__((target_version("rdm"))) unused_without_default(void); }; @@ -75,6 +75,13 @@ int bar() { // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver //. +// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: ret i32 1 +// +// // CHECK-LABEL: @_Z3fooi.default( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 @@ -82,6 +89,11 @@ int bar() { // CHECK-NEXT: ret i32 2 // // +// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 3 +// +// // CHECK-LABEL: @_Z3foov.default( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -189,6 +201,14 @@ int bar() { // CHECK-NEXT: ret i32 1 // // +// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// // CHECK-LABEL: @_Z3barv( // CHECK-NEXT: entry: // CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 @@ -250,26 +270,6 @@ int bar() { // CHECK-NEXT: ret ptr @_Z3foov.default // // -// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: ret i32 1 -// -// -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 -// -// -// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 0 -// -// // CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -312,16 +312,16 @@ int bar() { // CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default // //. -// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } -// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } // CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" } // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } //. diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 47dbeca206a9..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -483,14 +483,16 @@ void just_fine(void) {} __arm_locally_streaming __attribute__((target_version("sme2"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} +// expected-error@-1 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} +// expected-cpp-error@-2 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} __attribute__((target_version("default"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} void fmv_caller() { cannot_work_version(); cannot_work_clones(); just_fine(); - just_fine_locally_streaming(); + incompatible_locally_streaming(); } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index e2940c434c2f..cd5be459456e 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -68,13 +68,15 @@ int __attribute__((target_version(""))) unsup1(void) { return 1; } void __attribute__((target_version("crc32"))) unsup2(void) {} void __attribute__((target_version("default+fp16"))) koo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("default+default+default"))) loo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} //expected-error@+2 {{redefinition of 'redef'}} //expected-note@-2 {{previous definition is here}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} -int __attribute__((target_version("sm4"))) def(void); +int def(void); void __attribute__((target_version("dit"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); @@ -83,7 +85,6 @@ int bar() { // expected-error@+2 {{reference to overloaded function could not be resolved; did you mean to call it?}} // expected-note@-3 {{possible target for call}} ovl++; - // expected-error@+1 {{no matching function for call to 'nodef'}} nodef(); return def(); } @@ -92,8 +93,6 @@ int __attribute__((target_version("sha1"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -// expected-note@+1 {{function multiversioning caused by this declaration}} -int __attribute__((target_version("fcma"))) prot(); int __attribute__((target_version("pmull"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} @@ -104,6 +103,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +// expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } int cargs() { return unspec_args(); } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index 0bd710c4e282..b3385f043590 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -9,7 +9,6 @@ void __attribute__((target_version("rcpc3"))) no_def(void); void __attribute__((target_version("mops"))) no_def(void); void __attribute__((target_version("rdma"))) no_def(void); -// expected-error@+1 {{no matching function for call to 'no_def'}} void foo(void) { no_def(); } constexpr int __attribute__((target_version("sve2"))) diff_const(void) { return 1; } @@ -41,6 +40,7 @@ inline int __attribute__((target_version("sme"))) diff_inline(void) { return 1; int __attribute__((target_version("fp16"))) diff_inline(void) { return 2; } inline int __attribute__((target_version("sme"))) diff_inline1(void) { return 1; } +//expected-error@+1 {{multiversioned function declaration has a different inline specification}} int __attribute__((target_version("default"))) diff_inline1(void) { return 2; } int __attribute__((target_version("fcma"))) diff_type1(void) { return 1; } @@ -59,8 +59,7 @@ int __attribute__((target_version("sve2-sha3"))) diff_type3(void) noexcept(true) template int __attribute__((target_version("default"))) temp(T) { return 1; } template int __attribute__((target_version("simd"))) temp1(T) { return 1; } -// expected-error@+1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} -template int __attribute__((target_version("sha3"))) temp1(T) { return 2; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} extern "C" { int __attribute__((target_version("aes"))) extc(void) { return 1; } @@ -70,17 +69,23 @@ int __attribute__((target_version("lse"))) extc(void) { return 1; } auto __attribute__((target_version("default"))) ret1(void) { return 1; } auto __attribute__((target_version("dpb"))) ret2(void) { return 1; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deduced return types}} auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} __attribute__((target_version("sve-i8mm"))) ~Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support defaulted functions}} Cls &__attribute__((target_version("ssbs"))) operator=(Cls &&) = delete; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deleted functions}} virtual void __attribute__((target_version("default"))) vfunc(); virtual void __attribute__((target_version("sm4"))) vfunc1(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support virtual functions}} }; __attribute__((target_version("sha3"))) void Decl(); -- GitLab From 7434a6b96c68b6835a6de6e98199dcfc35981dd5 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 25 Mar 2024 10:50:08 +0100 Subject: [PATCH 304/933] [TableGen] Fix Linker Errors Fix linker errors after landing fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 --- llvm/utils/TableGen/Basic/CMakeLists.txt | 2 +- llvm/utils/TableGen/CMakeLists.txt | 2 +- llvm/utils/TableGen/Common/CMakeLists.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt index f2927d05c175..5a899e3b7c80 100644 --- a/llvm/utils/TableGen/Basic/CMakeLists.txt +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -8,7 +8,7 @@ set(LLVM_LINK_COMPONENTS TableGen ) -add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL +add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL CodeGenIntrinsics.cpp SDNodeProperties.cpp ) diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 14690329cabf..577aeded4be7 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -75,10 +75,10 @@ add_tablegen(llvm-tblgen LLVM X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp + $ $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt index 491d9bd2949d..0e985a083247 100644 --- a/llvm/utils/TableGen/Common/CMakeLists.txt +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -39,7 +39,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL vt_gen ) set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") -target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) # Users may include its headers as "Common/*.h" target_include_directories(LLVMTableGenCommon -- GitLab From 336bdf1a255571f8d894e8befe4be7e9141f7541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Mon, 25 Mar 2024 11:52:29 +0200 Subject: [PATCH 305/933] [verify-uselistorder] Fix a typo, fix linking This fixes a typo from 2ef612050844355906e4b67d892a00bbb58c41d6, which broke the build with errors like: ld.lld: error: undefined symbol: LoadBitcodeIntoNewDbgInforFormat >>> referenced by verify-uselistorder.cpp >>> tools/verify-uselistorder/CMakeFiles/verify-uselistorder.dir/verify-uselistorder.cpp.o:(main) >>> did you mean: LoadBitcodeIntoNewDbgInfoFormat >>> defined in: lib/libLLVMBitReader.a(BitcodeReader.cpp.o) collect2: error: ld returned 1 exit status --- llvm/tools/verify-uselistorder/verify-uselistorder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index 8b299c394e4e..cb07dede1d13 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,7 +68,7 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); -extern cl::opt LoadBitcodeIntoNewDbgInforFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -546,8 +546,8 @@ int main(int argc, char **argv) { "llvm tool to verify use-list order\n"); // Do not load bitcode into the new debug info format by default. - if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) - LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_FALSE; LLVMContext Context; SMDiagnostic Err; -- GitLab From dbfc38ed6b3f2a9be0b1a86b2a074aad69eb58a6 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Mon, 25 Mar 2024 18:57:53 +0900 Subject: [PATCH 306/933] [mlir][bufferization] Add `BufferOriginAnalysis` (#86461) This commit adds the `BufferOriginAnalysis`, which can be queried to check if two buffer SSA values originate from the same allocation. This new analysis is used in the buffer deallocation pass to fold away or simplify `bufferization.dealloc` ops more aggressively. The `BufferOriginAnalysis` is based on the `BufferViewFlowAnalysis`, which collects buffer SSA value "same buffer" dependencies. E.g., given IR such as: ``` %0 = memref.alloc() %1 = memref.subview %0 %2 = memref.subview %1 ``` The `BufferViewFlowAnalysis` will report the following "reverse" dependencies (`resolveReverse`) for `%2`: {`%2`, `%1`, `%0`}. I.e., all buffer SSA values in the reverse use-def chain that originate from the same allocation as `%2`. The `BufferOriginAnalysis` is built on top of that. It handles only simple cases at the moment and may conservatively return "unknown" around certain IR with branches, memref globals and function arguments. This analysis enables additional simplifications during `-buffer-deallocation-simplification`. In particular, "regular" scf.for loop nests, that yield buffers (or reallocations thereof) in the same order as they appear in the iter_args, are now handled much more efficiently. Such IR patterns are generated by the sparse compiler. --- .../Bufferization/IR/BufferizationOps.td | 1 + .../Transforms/BufferViewFlowAnalysis.h | 36 ++++ .../BufferDeallocationSimplification.cpp | 80 ++++----- .../Transforms/BufferViewFlowAnalysis.cpp | 160 ++++++++++++++++-- .../dealloc-loops.mlir | 86 ++++++++++ .../buffer-deallocation-simplification.mlir | 14 +- 6 files changed, 321 insertions(+), 56 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td index 9dc6afcaab31..4f609ddff9a4 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td @@ -10,6 +10,7 @@ #define BUFFERIZATION_OPS include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td" +include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizationBase.td" include "mlir/Interfaces/DestinationStyleOpInterface.td" diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 9e43265c5dfe..4015231c845d 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -53,6 +53,7 @@ public: /// /// Results in resolve(B) returning {B, C} ValueSetT resolve(Value value) const; + ValueSetT resolveReverse(Value value) const; /// Removes the given values from all alias sets. void remove(const SetVector &aliasValues); @@ -73,11 +74,46 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + ValueMapT reverseDependencies; /// A set of all SSA values that may be terminal buffers. DenseSet terminals; }; +/// An is-same-buffer analysis that checks if two SSA values belong to the same +/// buffer allocation or not. +class BufferOriginAnalysis { +public: + BufferOriginAnalysis(Operation *op); + + /// Return "true" if `v1` and `v2` originate from the same buffer allocation. + /// Return "false" if `v1` and `v2` originate from different allocations. + /// Return "nullopt" if we do not know for sure. + /// + /// Example 1: isSameAllocation(%0, %1) == true + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.subview %0 + /// ``` + /// + /// Example 2: isSameAllocation(%0, %1) == false + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// ``` + /// + /// Example 3: isSameAllocation(%0, %2) == nullopt + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// %2 = arith.select %c, %0, %1 + /// ``` + std::optional isSameAllocation(Value v1, Value v2); + +private: + BufferViewFlowAnalysis analysis; +}; + } // namespace mlir #endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERVIEWFLOWANALYSIS_H diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp index e30779868b47..954485cfede3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Analysis/AliasAnalysis.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" #include "mlir/Dialect/Bufferization/Transforms/Passes.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" @@ -34,6 +34,14 @@ using namespace mlir::bufferization; // Helpers //===----------------------------------------------------------------------===// +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, ValueRange memrefs, ValueRange conditions, @@ -49,14 +57,6 @@ static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, return success(); } -/// Given a memref value, return the "base" value by skipping over all -/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. -static Value getViewBase(Value value) { - while (auto viewLikeOp = value.getDefiningOp()) - value = viewLikeOp.getViewSource(); - return value; -} - /// Return "true" if the given values are guaranteed to be different (and /// non-aliasing) allocations based on the fact that one value is the result /// of an allocation and the other value is a block argument of a parent block. @@ -80,12 +80,14 @@ static bool distinctAllocAndBlockArgument(Value v1, Value v2) { /// Checks if `memref` may potentially alias a MemRef in `otherList`. It is /// often a requirement of optimization patterns that there cannot be any /// aliasing memref in order to perform the desired simplification. -static bool potentiallyAliasesMemref(AliasAnalysis &analysis, +static bool potentiallyAliasesMemref(BufferOriginAnalysis &analysis, ValueRange otherList, Value memref) { for (auto other : otherList) { if (distinctAllocAndBlockArgument(other, memref)) continue; - if (!analysis.alias(other, memref).isNo()) + std::optional analysisResult = + analysis.isSameAllocation(other, memref); + if (!analysisResult.has_value() || analysisResult == true) return true; } return false; @@ -129,8 +131,8 @@ namespace { struct RemoveDeallocMemrefsContainedInRetained : public OpRewritePattern { RemoveDeallocMemrefsContainedInRetained(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} /// The passed 'memref' must not have a may-alias relation to any retained /// memref, and at least one must-alias relation. If there is no must-aliasing @@ -147,10 +149,11 @@ struct RemoveDeallocMemrefsContainedInRetained // deallocated in some situations and can thus not be dropped). bool atLeastOneMustAlias = false; for (Value retained : deallocOp.getRetained()) { - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMay()) + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (!analysisResult.has_value()) return failure(); - if (analysisResult.isMust() || analysisResult.isPartial()) + if (analysisResult == true) atLeastOneMustAlias = true; } if (!atLeastOneMustAlias) @@ -161,8 +164,9 @@ struct RemoveDeallocMemrefsContainedInRetained // we can remove that operand later on. for (auto [i, retained] : llvm::enumerate(deallocOp.getRetained())) { Value updatedCondition = deallocOp.getUpdatedConditions()[i]; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { auto disjunction = rewriter.create( deallocOp.getLoc(), updatedCondition, cond); rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(), @@ -206,7 +210,7 @@ struct RemoveDeallocMemrefsContainedInRetained } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Remove memrefs from the `retained` list which are guaranteed to not alias @@ -228,15 +232,15 @@ private: struct RemoveRetainedMemrefsGuaranteedToNotAlias : public OpRewritePattern { RemoveRetainedMemrefsGuaranteedToNotAlias(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { SmallVector newRetainedMemrefs, replacements; for (auto retainedMemref : deallocOp.getRetained()) { - if (potentiallyAliasesMemref(aliasAnalysis, deallocOp.getMemrefs(), + if (potentiallyAliasesMemref(analysis, deallocOp.getMemrefs(), retainedMemref)) { newRetainedMemrefs.push_back(retainedMemref); replacements.push_back({}); @@ -264,7 +268,7 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Split off memrefs to separate dealloc operations to reduce the number of @@ -297,8 +301,8 @@ private: struct SplitDeallocWhenNotAliasingAnyOther : public OpRewritePattern { SplitDeallocWhenNotAliasingAnyOther(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -314,7 +318,7 @@ struct SplitDeallocWhenNotAliasingAnyOther SmallVector otherMemrefs(deallocOp.getMemrefs()); otherMemrefs.erase(otherMemrefs.begin() + i); // Check if `memref` can split off into a separate bufferization.dealloc. - if (potentiallyAliasesMemref(aliasAnalysis, otherMemrefs, memref)) { + if (potentiallyAliasesMemref(analysis, otherMemrefs, memref)) { // `memref` alias with other memrefs, do not split off. remainingMemrefs.push_back(memref); remainingConditions.push_back(cond); @@ -352,7 +356,7 @@ struct SplitDeallocWhenNotAliasingAnyOther } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Check for every retained memref if a must-aliasing memref exists in the @@ -381,8 +385,8 @@ private: struct RetainedMemrefAliasingAlwaysDeallocatedMemref : public OpRewritePattern { RetainedMemrefAliasingAlwaysDeallocatedMemref(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -396,8 +400,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!matchPattern(cond, m_One())) continue; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -411,10 +416,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!extractOp) continue; - AliasResult extractAnalysisResult = - aliasAnalysis.alias(retained, extractOp.getOperand()); - if (extractAnalysisResult.isMust() || - extractAnalysisResult.isPartial()) { + std::optional extractAnalysisResult = + analysis.isSameAllocation(retained, extractOp.getOperand()); + if (extractAnalysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -434,7 +438,7 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; } // namespace @@ -452,13 +456,13 @@ struct BufferDeallocationSimplificationPass : public bufferization::impl::BufferDeallocationSimplificationBase< BufferDeallocationSimplificationPass> { void runOnOperation() override { - AliasAnalysis &aliasAnalysis = getAnalysis(); + BufferOriginAnalysis analysis(getOperation()); RewritePatternSet patterns(&getContext()); patterns.add(&getContext(), - aliasAnalysis); + analysis); populateDeallocOpCanonicalizationPatterns(patterns, &getContext()); if (failed( diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 9a36057425f3..72f47b8b468e 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -19,22 +19,23 @@ using namespace mlir; using namespace mlir::bufferization; +//===----------------------------------------------------------------------===// +// BufferViewFlowAnalysis +//===----------------------------------------------------------------------===// + /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } -/// Find all immediate and indirect dependent buffers this value could -/// potentially have. Note that the resulting set will also contain the value -/// provided as it is a dependent alias of itself. -BufferViewFlowAnalysis::ValueSetT -BufferViewFlowAnalysis::resolve(Value rootValue) const { - ValueSetT result; +static BufferViewFlowAnalysis::ValueSetT +resolveValues(const BufferViewFlowAnalysis::ValueMapT &map, Value value) { + BufferViewFlowAnalysis::ValueSetT result; SmallVector queue; - queue.push_back(rootValue); + queue.push_back(value); while (!queue.empty()) { Value currentValue = queue.pop_back_val(); if (result.insert(currentValue).second) { - auto it = dependencies.find(currentValue); - if (it != dependencies.end()) { + auto it = map.find(currentValue); + if (it != map.end()) { for (Value aliasValue : it->second) queue.push_back(aliasValue); } @@ -43,6 +44,19 @@ BufferViewFlowAnalysis::resolve(Value rootValue) const { return result; } +/// Find all immediate and indirect dependent buffers this value could +/// potentially have. Note that the resulting set will also contain the value +/// provided as it is a dependent alias of itself. +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolve(Value rootValue) const { + return resolveValues(dependencies, rootValue); +} + +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolveReverse(Value rootValue) const { + return resolveValues(reverseDependencies, rootValue); +} + /// Removes the given values from all alias sets. void BufferViewFlowAnalysis::remove(const SetVector &aliasValues) { for (auto &entry : dependencies) @@ -69,8 +83,10 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip_equal(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) { this->dependencies[value].insert(dep); + this->reverseDependencies[dep].insert(value); + } }; // Mark all buffer results and buffer region entry block arguments of the @@ -188,3 +204,127 @@ bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { assert(isa(value.getType()) && "expected memref"); return terminals.contains(value); } + +//===----------------------------------------------------------------------===// +// BufferOriginAnalysis +//===----------------------------------------------------------------------===// + +/// Return "true" if the given value is the result of a memory allocation. +static bool hasAllocateSideEffect(Value v) { + Operation *op = v.getDefiningOp(); + if (!op) + return false; + return hasEffect(op, v); +} + +/// Return "true" if the given value is a function block argument. +static bool isFunctionArgument(Value v) { + auto bbArg = dyn_cast(v); + if (!bbArg) + return false; + Block *b = bbArg.getOwner(); + auto funcOp = dyn_cast(b->getParentOp()); + if (!funcOp) + return false; + return bbArg.getOwner() == &funcOp.getFunctionBody().front(); +} + +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + +BufferOriginAnalysis::BufferOriginAnalysis(Operation *op) : analysis(op) {} + +std::optional BufferOriginAnalysis::isSameAllocation(Value v1, Value v2) { + assert(isa(v1.getType()) && "expected buffer"); + assert(isa(v2.getType()) && "expected buffer"); + + // Skip over all view-like ops. + v1 = getViewBase(v1); + v2 = getViewBase(v2); + + // Fast path: If both buffers are the same SSA value, we can be sure that + // they originate from the same allocation. + if (v1 == v2) + return true; + + // Compute the SSA values from which the buffers `v1` and `v2` originate. + SmallPtrSet origin1 = analysis.resolveReverse(v1); + SmallPtrSet origin2 = analysis.resolveReverse(v2); + + // Originating buffers are "terminal" if they could not be traced back any + // further by the `BufferViewFlowAnalysis`. Examples of terminal buffers: + // - function block arguments + // - values defined by allocation ops such as "memref.alloc" + // - values defined by ops that are unknown to the buffer view flow analysis + // - values that are marked as "terminal" in the `BufferViewFlowOpInterface` + SmallPtrSet terminal1, terminal2; + + // While gathering terminal buffers, keep track of whether all terminal + // buffers are newly allocated buffer or function entry arguments. + bool allAllocs1 = true, allAllocs2 = true; + bool allAllocsOrFuncEntryArgs1 = true, allAllocsOrFuncEntryArgs2 = true; + + // Helper function that gathers terminal buffers among `origin`. + auto gatherTerminalBuffers = [this](const SmallPtrSet &origin, + SmallPtrSet &terminal, + bool &allAllocs, + bool &allAllocsOrFuncEntryArgs) { + for (Value v : origin) { + if (isa(v.getType()) && analysis.mayBeTerminalBuffer(v)) { + terminal.insert(v); + allAllocs &= hasAllocateSideEffect(v); + allAllocsOrFuncEntryArgs &= + isFunctionArgument(v) || hasAllocateSideEffect(v); + } + } + assert(!terminal.empty() && "expected non-empty terminal set"); + }; + + // Gather terminal buffers for `v1` and `v2`. + gatherTerminalBuffers(origin1, terminal1, allAllocs1, + allAllocsOrFuncEntryArgs1); + gatherTerminalBuffers(origin2, terminal2, allAllocs2, + allAllocsOrFuncEntryArgs2); + + // If both `v1` and `v2` have a single matching terminal buffer, they are + // guaranteed to originate from the same buffer allocation. + if (llvm::hasSingleElement(terminal1) && llvm::hasSingleElement(terminal2) && + *terminal1.begin() == *terminal2.begin()) + return true; + + // At least one of the two values has multiple terminals. + + // Check if there is overlap between the terminal buffers of `v1` and `v2`. + bool distinctTerminalSets = true; + for (Value v : terminal1) + distinctTerminalSets &= !terminal2.contains(v); + // If there is overlap between the terminal buffers of `v1` and `v2`, we + // cannot make an accurate decision without further analysis. + if (!distinctTerminalSets) + return std::nullopt; + + // If `v1` originates from only allocs, and `v2` is guaranteed to originate + // from different allocations (that is guaranteed if `v2` originates from + // only distinct allocs or function entry arguments), we can be sure that + // `v1` and `v2` originate from different allocations. The same argument can + // be made when swapping `v1` and `v2`. + bool isolatedAlloc1 = allAllocs1 && (allAllocs2 || allAllocsOrFuncEntryArgs2); + bool isolatedAlloc2 = (allAllocs1 || allAllocsOrFuncEntryArgs1) && allAllocs2; + if (isolatedAlloc1 || isolatedAlloc2) + return false; + + // Otherwise: We do not know whether `v1` and `v2` originate from the same + // allocation or not. + // TODO: Function arguments are currently handled conservatively. We assume + // that they could be the same allocation. + // TODO: Terminals other than allocations and function arguments are + // currently handled conservatively. We assume that they could be the same + // allocation. E.g., we currently return "nullopt" for values that originate + // from different "memref.get_global" ops (with different symbols). + return std::nullopt; +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir new file mode 100644 index 000000000000..53b28c3aab6f --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir @@ -0,0 +1,86 @@ +// RUN: mlir-opt %s -expand-realloc="emit-deallocs=false" -ownership-based-buffer-deallocation="private-function-dynamic-ownership=true" -canonicalize -buffer-deallocation-simplification | FileCheck %s + +// A function that reallocates two buffer inside of a loop. The simplification +// pass should be able to figure out that the iter_args are always originating +// from different allocations. IR like this one appears in the sparse compiler. + +// CHECK-LABEL: func private @loop_with_realloc( +func.func private @loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + // CHECK: %[[m2:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m2 = memref.realloc %arg0(%s2) : memref to memref + // CHECK: %[[m3:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m3 = memref.realloc %arg1(%s2) : memref to memref + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg0]] + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg1]] + // CHECK: %[[d0:.*]] = bufferization.dealloc (%[[base0]] : memref) if (%[[o0]]) retain (%[[m2]]#0 : memref) + // CHECK: %[[d1:.*]] = bufferization.dealloc (%[[base1]] : memref) if (%[[o1]]) retain (%[[m3]]#0 : memref) + // CHECK-DAG: %[[o2:.*]] = arith.ori %[[d0]], %[[m2]]#1 + // CHECK-DAG: %[[o3:.*]] = arith.ori %[[d1]], %[[m3]]#1 + // CHECK: scf.yield %[[m2]]#0, %[[m3]]#0, %[[o2]], %[[o3]] + scf.yield %m2, %m3 : memref, memref + } + + // CHECK: %[[d2:.*]] = bufferization.dealloc (%[[m0]] : memref) if (%[[true]]) retain (%[[r]]#0 : memref) + // CHECK: %[[d3:.*]] = bufferization.dealloc (%[[m1]] : memref) if (%[[true]]) retain (%[[r]]#1 : memref) + // CHECK-DAG: %[[or0:.*]] = arith.ori %[[d2]], %[[r]]#2 + // CHECK-DAG: %[[or1:.*]] = arith.ori %[[d3]], %[[r]]#3 + // CHECK: return %[[r]]#0, %[[r]]#1, %[[or0]], %[[or1]] + return %r0, %r1 : memref, memref +} + +// ----- + +// The yielded values of the loop are swapped. Therefore, the +// bufferization.dealloc before the func.return can now longer be split, +// because %r0 could originate from either %m0 and %m1 (same for %r1). + +// CHECK-LABEL: func private @swapping_loop_with_realloc( +func.func private @swapping_loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + %m2 = memref.realloc %arg0(%s2) : memref to memref + %m3 = memref.realloc %arg1(%s2) : memref to memref + scf.yield %m3, %m2 : memref, memref + } + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#0 + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#1 + // CHECK: %[[d:.*]]:2 = bufferization.dealloc (%[[m0]], %[[m1]], %[[base0]], %[[base1]] : {{.*}}) if (%[[true]], %[[true]], %[[r]]#2, %[[r]]#3) retain (%[[r]]#0, %[[r]]#1 : {{.*}}) + // CHECK: return %[[r]]#0, %[[r]]#1, %[[d]]#0, %[[d]]#1 + return %r0, %r1 : memref, memref +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir index eee69acbe821..b40a17cf800b 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir @@ -92,15 +92,13 @@ func.func @dealloc_split_when_no_other_aliasing(%arg0: i1, %arg1: memref<2xi32>, // CHECK-NEXT: [[ALLOC0:%.+]] = memref.alloc( // CHECK-NEXT: [[ALLOC1:%.+]] = memref.alloc( // CHECK-NEXT: [[V0:%.+]] = arith.select{{.*}}[[ALLOC0]], [[ALLOC1]] : -// COM: there is only one value in the retained list because the -// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here and -// COM: removes %arg1 from the list. In the second dealloc, this does not apply -// COM: because function arguments are assumed potentially alias (even if the -// COM: types don't exactly match). +// COM: there is only one value in the retained lists because the +// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here: +// COM: - %alloc is guaranteed to not alias with %arg1. +// COM: - %arg2 is guaranteed to not alias with %0. // CHECK-NEXT: [[V1:%.+]] = bufferization.dealloc ([[ALLOC0]] : memref<2xi32>) if ([[ARG0]]) retain ([[V0]] : memref<2xi32>) -// CHECK-NEXT: [[V2:%.+]]:2 = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]], [[V0]] : memref<2xi32>, memref<2xi32>) -// CHECK-NEXT: [[V3:%.+]] = arith.ori [[V1]], [[V2]]#1 -// CHECK-NEXT: return [[V2]]#0, [[V3]] : +// CHECK-NEXT: [[V2:%.+]] = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]] : memref<2xi32>) +// CHECK-NEXT: return [[V2]], [[V1]] : // ----- -- GitLab From 94a550dab26c4b30a187dd8e3ce431e0f915923b Mon Sep 17 00:00:00 2001 From: Mariusz Sikora Date: Mon, 25 Mar 2024 11:00:59 +0100 Subject: [PATCH 307/933] [AMDGPU][NFC] Rename Feature GFX11FullVGPRs to 1_5xVGPRs (#86468) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 14 ++++++++------ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index c877658cd38e..37dcfef3b2a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -831,10 +831,12 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard", "Does not need SW waitstates" >; -def FeatureGFX11FullVGPRs : SubtargetFeature<"gfx11-full-vgprs", - "HasGFX11FullVGPRs", +// Allocate 1536 VGPRs for wave32 and 768 VGPRs for wave64 +// with allocation granularity 24 for wave32 and 12 for wave64 +def Feature1_5xVGPRs : SubtargetFeature<"allocate1_5xvgprs", + "Has1_5xVGPRs", "true", - "GFX11 with 50% more physical VGPRs and 50% larger allocation granule than GFX10" + "Has 50% more physical VGPRs and 50% larger allocation granule" >; @@ -1491,12 +1493,12 @@ def FeatureISAVersion11_0_Common : FeatureSet< def FeatureISAVersion11_0_0 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs, + [Feature1_5xVGPRs, FeatureUserSGPRInit16Bug])>; def FeatureISAVersion11_0_1 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs])>; + [Feature1_5xVGPRs])>; def FeatureISAVersion11_0_2 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, @@ -1517,7 +1519,7 @@ def FeatureISAVersion11_5_1 : FeatureSet< [FeatureSALUFloatInsts, FeatureDPPSrc1SGPR, FeatureVGPRSingleUseHintInsts, - FeatureGFX11FullVGPRs])>; + Feature1_5xVGPRs])>; def FeatureISAVersion12 : FeatureSet< [FeatureGFX12, diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index ca51da659c33..4da10beabe31 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -223,7 +223,7 @@ protected: bool HasImageStoreD16Bug = false; bool HasImageGather4D16Bug = false; bool HasMSAALoadDstSelBug = false; - bool HasGFX11FullVGPRs = false; + bool Has1_5xVGPRs = false; bool HasMADIntraFwdBug = false; bool HasVOPDInsts = false; bool HasVALUTransUseHazard = false; @@ -1202,7 +1202,7 @@ public: /// target. bool hasNullExportTarget() const { return !GFX11Insts; } - bool hasGFX11FullVGPRs() const { return HasGFX11FullVGPRs; } + bool has1_5xVGPRs() const { return Has1_5xVGPRs; } bool hasVOPDInsts() const { return HasVOPDInsts; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index a90dc32d396f..7bb84d78442b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1087,7 +1087,7 @@ unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, *EnableWavefrontSize32 : STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 24 : 12; if (hasGFX10_3Insts(*STI)) @@ -1114,7 +1114,7 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { if (!isGFX10Plus(*STI)) return 256; bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 1536 : 768; return IsWave32 ? 1024 : 512; } -- GitLab From 37785fedabd8fa752129ef5bac3462311af91c35 Mon Sep 17 00:00:00 2001 From: Discookie Date: Mon, 25 Mar 2024 10:08:56 +0000 Subject: [PATCH 308/933] [clang][analyzer] Bring cplusplus.ArrayDelete out of alpha (#83985) The checker finds a type of undefined behavior, where if the type of a pointer to an object-array is different from the objects' underlying type, calling `delete[]` is undefined, as the size of the two objects might be different. The checker has been in alpha for a while now, it is a simple checker that causes no crashes, and considering the severity of the issue, it has a low result-count on open-source projects (in my last test-run on my usual projects, it had 0 results). This commit cleans up the documentation and adds docs for the limitation related to tracking through references, in addition to moving it to `cplusplus`. --------- Co-authored-by: Balazs Benics Co-authored-by: whisperity --- clang/docs/analyzer/checkers.rst | 69 ++++++++++++------- .../clang/StaticAnalyzer/Checkers/Checkers.td | 10 +-- .../Checkers/CXXDeleteChecker.cpp | 4 +- clang/test/Analysis/ArrayDelete.cpp | 2 +- clang/www/analyzer/alpha_checks.html | 20 ------ clang/www/analyzer/available_checks.html | 27 ++++++++ 6 files changed, 80 insertions(+), 52 deletions(-) diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst index fe2115149142..66da1c7b35f2 100644 --- a/clang/docs/analyzer/checkers.rst +++ b/clang/docs/analyzer/checkers.rst @@ -340,6 +340,51 @@ cplusplus C++ Checkers. +.. _cplusplus-ArrayDelete: + +cplusplus.ArrayDelete (C++) +""""""""""""""""""""""""""" + +Reports destructions of arrays of polymorphic objects that are destructed as +their base class. If the dynamic type of the array is different from its static +type, calling `delete[]` is undefined. + +This checker corresponds to the SEI CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. + +.. code-block:: cpp + + class Base { + public: + virtual ~Base() {} + }; + class Derived : public Base {}; + + Base *create() { + Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here + return x; + } + + void foo() { + Base *x = create(); + delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + +**Limitations** + +The checker does not emit note tags when casting to and from reference types, +even though the pointer values are tracked across references. + +.. code-block:: cpp + + void foo() { + Derived *d = new Derived[10]; + Derived &dref = *d; + + Base &bref = static_cast(dref); // no note + Base *b = &bref; + delete[] b; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + .. _cplusplus-InnerPointer: cplusplus.InnerPointer (C++) @@ -2139,30 +2184,6 @@ Either the comparison is useless or there is division by zero. alpha.cplusplus ^^^^^^^^^^^^^^^ -.. _alpha-cplusplus-ArrayDelete: - -alpha.cplusplus.ArrayDelete (C++) -""""""""""""""""""""""""""""""""" -Reports destructions of arrays of polymorphic objects that are destructed as their base class. -This checker corresponds to the CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. - -.. code-block:: cpp - - class Base { - virtual ~Base() {} - }; - class Derived : public Base {} - - Base *create() { - Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here - return x; - } - - void foo() { - Base *x = create(); - delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined - } - .. _alpha-cplusplus-DeleteWithNonVirtualDtor: alpha.cplusplus.DeleteWithNonVirtualDtor (C++) diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td index 686e5e99f4a6..bf46766d44b3 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td @@ -622,6 +622,11 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">, let ParentPackage = Cplusplus in { +def ArrayDeleteChecker : Checker<"ArrayDelete">, + HelpText<"Reports destructions of arrays of polymorphic objects that are " + "destructed as their base class.">, + Documentation; + def InnerPointerChecker : Checker<"InnerPointer">, HelpText<"Check for inner pointers of C++ containers used after " "re/deallocation">, @@ -777,11 +782,6 @@ def ContainerModeling : Checker<"ContainerModeling">, Documentation, Hidden; -def CXXArrayDeleteChecker : Checker<"ArrayDelete">, - HelpText<"Reports destructions of arrays of polymorphic objects that are " - "destructed as their base class.">, - Documentation; - def DeleteWithNonVirtualDtorChecker : Checker<"DeleteWithNonVirtualDtor">, HelpText<"Reports destructions of polymorphic objects with a non-virtual " "destructor in their base class">, diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp index b4dee1e300e8..1b1226a7f1a7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp @@ -220,11 +220,11 @@ CXXDeleteChecker::PtrCastVisitor::VisitNode(const ExplodedNode *N, /*addPosRange=*/true); } -void ento::registerCXXArrayDeleteChecker(CheckerManager &mgr) { +void ento::registerArrayDeleteChecker(CheckerManager &mgr) { mgr.registerChecker(); } -bool ento::shouldRegisterCXXArrayDeleteChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterArrayDeleteChecker(const CheckerManager &mgr) { return true; } diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp index 3b8d49552376..6887e0a35fb8 100644 --- a/clang/test/Analysis/ArrayDelete.cpp +++ b/clang/test/Analysis/ArrayDelete.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s +// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s struct Base { virtual ~Base() = default; diff --git a/clang/www/analyzer/alpha_checks.html b/clang/www/analyzer/alpha_checks.html index 7bbe4a20288f..f040d1957b0f 100644 --- a/clang/www/analyzer/alpha_checks.html +++ b/clang/www/analyzer/alpha_checks.html @@ -307,26 +307,6 @@ void test(int x) { - - - -