From b3fe27f2be0585d5d2ad46f96956ccfd76ca003e Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Sun, 24 Mar 2024 16:14:56 +0100 Subject: [PATCH 001/343] [InstCombine] Copy flags of extractelement for extelt -> icmp combine (#86366) Fixes #86164 --- llvm/include/llvm/IR/InstrTypes.h | 11 +++++++++++ llvm/lib/IR/Instructions.cpp | 10 ++++++++++ .../Transforms/InstCombine/InstCombineVectorOps.cpp | 4 +++- llvm/test/Transforms/InstCombine/scalarization.ll | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index e8c2cba8418d..d2b33fe9c651 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1058,6 +1058,17 @@ public: static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); + /// Construct a compare instruction, given the opcode, the predicate, + /// the two operands and the instruction to copy the flags from. Optionally + /// (if InstBefore is specified) insert the instruction into a BasicBlock + /// right before the specified instruction. The specified Instruction is + /// allowed to be a dereferenced end iterator. Create a CmpInst + static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name = "", + Instruction *InsertBefore = nullptr); + /// Get the opcode casted to the right type OtherOps getOpcode() const { return static_cast(Instruction::getOpcode()); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 494d50f89e37..8ca211e6e2e7 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -4623,6 +4623,16 @@ CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, S1, S2, Name); } +CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name, + Instruction *InsertBefore) { + CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore); + Inst->copyIRFlags(FlagsSource); + return Inst; +} + void CmpInst::swapOperands() { if (ICmpInst *IC = dyn_cast(this)) IC->swapOperands(); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c7f4fb17648c..99f1f8eb34bb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -487,7 +487,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index) Value *E0 = Builder.CreateExtractElement(X, Index); Value *E1 = Builder.CreateExtractElement(Y, Index); - return CmpInst::Create(cast(SrcVec)->getOpcode(), Pred, E0, E1); + CmpInst *SrcCmpInst = cast(SrcVec); + return CmpInst::CreateWithCopiedFlags(SrcCmpInst->getOpcode(), Pred, E0, E1, + SrcCmpInst); } if (auto *I = dyn_cast(SrcVec)) { diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll index fe6dc526bd50..7e645ef7e883 100644 --- a/llvm/test/Transforms/InstCombine/scalarization.ll +++ b/llvm/test/Transforms/InstCombine/scalarization.ll @@ -341,6 +341,17 @@ define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) { ret i1 %ext } +define i1 @extractelt_vector_fcmp_copy_flags(<4 x float> %x) { +; CHECK-LABEL: @extractelt_vector_fcmp_copy_flags( +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = fcmp nsz arcp oeq float [[TMP1]], 0.000000e+00 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %cmp = fcmp nsz arcp oeq <4 x float> %x, zeroinitializer + %r = extractelement <4 x i1> %cmp, i32 2 + ret i1 %r +} + define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) { ; ; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use( -- GitLab From 9632e1515c93453efc39752b1c9f32aedd358fbc Mon Sep 17 00:00:00 2001 From: houndlord <45179481+houndlord@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:33:16 +0100 Subject: [PATCH 002/343] Match fixed width ISD::AVGFLOORS + ISD::AVGCEILS patterns (#86222) --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++++++++++----- llvm/test/CodeGen/AArch64/hadd-combine.ll | 24 ++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index e27a8bb8fdac..05b4ce3aaa2c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2529,20 +2529,28 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); } -// Attempt to form avgceilu(A, B) from (A | B) - ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGCEILU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) { - SDValue A, B; if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) { + if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B); + } + } return SDValue(); } @@ -2837,20 +2845,29 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { return SDValue(); } -// Attempt to form avgflooru(A, B) from (A & B) + ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGFLOORU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) { - SDValue A, B; if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) { + if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B); + } + } + return SDValue(); } @@ -2869,8 +2886,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGFLOORU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGFLOORU(N, DAG)) + // Try to match AVGFLOOR fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG)) return V; // fold (a+b) -> (a|b) iff a and b share no bits. @@ -3868,8 +3885,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGCEILU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGCEILU(N, DAG)) + // Try to match AVGCEIL fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG)) return V; if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N))) diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll index e12502980790..491bf40ea4aa 100644 --- a/llvm/test/CodeGen/AArch64/hadd-combine.ll +++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll @@ -341,6 +341,18 @@ define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: srhadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %or = or <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = sub <8 x i16> %or, %srl + ret <8 x i16> %res +} + define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) { ; CHECK-LABEL: rhaddu_base: ; CHECK: // %bb.0: @@ -879,6 +891,18 @@ define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: shadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %and = and <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = add <8 x i16> %and, %srl + ret <8 x i16> %res +} + declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>) declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>) declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>) -- GitLab From 48048051323d5dd74057dc5f32df8c3c323afcd5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Sun, 24 Mar 2024 11:43:15 -0600 Subject: [PATCH 003/343] [lld/ELF][X86] Respect outSecOff when checking if GOTPCREL can be relaxed (#86334) The existing implementation didn't handle when the input text section was some offset from the output section. This resulted in an assert in relaxGot() with an lld built with asserts for some large binaries, or even worse, a silently broken binary with an lld without asserts. --- lld/ELF/Arch/X86_64.cpp | 7 ++++--- lld/test/ELF/x86-64-gotpc-relax-too-far.s | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index de459013595f..a85bf3aa0c09 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -328,9 +328,10 @@ bool X86_64::relaxOnce(int pass) const { if (rel.expr != R_RELAX_GOT_PC) continue; - uint64_t v = sec->getRelocTargetVA( - sec->file, rel.type, rel.addend, - sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr); + uint64_t v = sec->getRelocTargetVA(sec->file, rel.type, rel.addend, + sec->getOutputSection()->addr + + sec->outSecOff + rel.offset, + *rel.sym, rel.expr); if (isInt<32>(v)) continue; if (rel.sym->auxIdx == 0) { diff --git a/lld/test/ELF/x86-64-gotpc-relax-too-far.s b/lld/test/ELF/x86-64-gotpc-relax-too-far.s index 74aa6d8f65a0..ba41faab67de 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-too-far.s +++ b/lld/test/ELF/x86-64-gotpc-relax-too-far.s @@ -5,7 +5,10 @@ # RUN: llvm-objdump --no-print-imm-hex -d %t/bin | FileCheck --check-prefix=DISASM %s # RUN: llvm-readelf -S %t/bin | FileCheck --check-prefixes=GOT %s # RUN: ld.lld -T %t/lds2 %t/a.o -o %t/bin2 -# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=UNNECESSARY-GOT %s +# RUN: llvm-objdump --no-print-imm-hex -d %t/bin2 | FileCheck --check-prefix=DISASM %s +# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=GOT %s +# RUN: ld.lld -T %t/lds3 %t/a.o -o %t/bin3 +# RUN: llvm-readelf -S %t/bin3 | FileCheck --check-prefixes=UNNECESSARY-GOT %s # DISASM: <_foo>: # DISASM-NEXT: movl 2097146(%rip), %eax @@ -47,6 +50,13 @@ SECTIONS { data 0x80200000 : { *(data) } } #--- lds2 +SECTIONS { + .text.foo 0x100000 : { *(.text.foo) } + .text 0x1ff000 : { . = . + 0x1000 ; *(.text) } + .got 0x300000 : { *(.got) } + data 0x80200000 : { *(data) } +} +#--- lds3 SECTIONS { .text.foo 0x100000 : { *(.text.foo) } .text 0x200000 : { *(.text) } -- GitLab From d4a4585165f5c6ca8c42920b70e1b47696ff1172 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 24 Mar 2024 19:52:19 +0100 Subject: [PATCH 004/343] [clang-tidy] Ignore expresions in unevaluated context in bugprone-inc-dec-in-conditions (#85849) Skip checking for references to variable in unevaluated context, like decltype, static_assert and so on. Closes #85838 --- .../clang-tidy/bugprone/IncDecInConditionsCheck.cpp | 8 +++++++- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/bugprone/inc-dec-in-conditions.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index 16f43128d55e..9b3b01eb0268 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -31,6 +31,10 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), cxxOperatorCallExpr(isComparisonOperator()))); + auto IsInUnevaluatedContext = + expr(anyOf(hasAncestor(expr(matchers::hasUnevaluatedContext())), + hasAncestor(typeLoc()))); + Finder->addMatcher( expr( OperatorMatcher, unless(isExpansionInSystemHeader()), @@ -42,12 +46,14 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { cxxOperatorCallExpr( isPrePostOperator(), hasUnaryOperand(expr().bind("operand")))), + unless(IsInUnevaluatedContext), hasAncestor( expr(equalsBoundNode("parent"), hasDescendant( expr(unless(equalsBoundNode("operand")), matchers::isStatementIdenticalToBoundNode( - "operand")) + "operand"), + unless(IsInUnevaluatedContext)) .bind("second"))))) .bind("operator"))), this); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index a604e9276668..2392ccaf6575 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -139,6 +139,10 @@ Changes in existing checks ` check by detecting side effect from calling a method with non-const reference parameters. +- Improved :doc:`bugprone-inc-dec-in-conditions + ` check to ignore code + within unevaluated contexts, such as ``decltype``. + - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion ` check by eliminating false positives resulting from direct usage of bitwise operators diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp index 82af039973c3..91de013138f0 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp @@ -68,3 +68,13 @@ bool doubleCheck(Container x) { // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: decrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] // CHECK-MESSAGES: :[[@LINE-2]]:31: warning: incrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] } + +namespace PR85838 { + void test() + { + auto foo = 0; + auto bar = 0; + if (++foo < static_cast(bar)) {} + if (static_cast(bar) < foo) {} + } +} -- GitLab From 909ea28ac60760a0e9b39369b2f54a4e8f1daec9 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 14:29:23 -0500 Subject: [PATCH 005/343] [Libomptarget] Specificall add LLVM include dirs in plugins --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 0420d0e6f1f8..b84f3d7b137c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -60,6 +60,7 @@ target_link_options(PluginCommon PUBLIC ${offload_link_flags}) target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_LLVM_INCLUDE_DIRS} ${LIBOMPTARGET_INCLUDE_DIR} ) -- GitLab From 488a18738f5c275093bcc5459da69e9b4c9de074 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sun, 24 Mar 2024 10:41:21 -0500 Subject: [PATCH 006/343] [JITLink][ELF] Improve ELF section start/end symbol handling. This commit adds section start and stop symbol handling to ELF/aarch64, and fixes the section symbol prefixes (using `__start_` and `__stop_`, rather than `__start` and `__end`). It also adds a testcase for handling of these symbols. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 2 +- .../DefineExternalSectionStartAndEndSymbols.h | 19 ++++++++ .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 7 +++ .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 18 -------- .../ELF_section_start_and_stop_symbols.s | 43 +++++++++++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 30a9383dc0bc..4ce0c8d9fd17 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -567,7 +567,7 @@ public: orc::ExecutorAddrDiff getOffset() const { return Offset; } void setOffset(orc::ExecutorAddrDiff NewOffset) { - assert(NewOffset < getBlock().getSize() && "Offset out of range"); + assert(NewOffset <= getBlock().getSize() && "Offset out of range"); Offset = NewOffset; } diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 159880e4b152..aef441d093a2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -108,6 +108,25 @@ createDefineExternalSectionStartAndEndSymbolsPass( std::forward(F)); } +/// ELF section start/end symbol detection. +inline SectionRangeSymbolDesc +identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { + constexpr StringRef StartSymbolPrefix = "__start_"; + constexpr StringRef EndSymbolPrefix = "__stop_"; + + auto SymName = Sym.getName(); + if (SymName.starts_with(StartSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) + return {*Sec, true}; + } else if (SymName.starts_with(EndSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) + return {*Sec, false}; + } + return {}; +} + } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index f17b2c626ac2..4f64d83162b2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -20,6 +20,8 @@ #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" +#include "DefineExternalSectionStartAndEndSymbols.h" + #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -611,6 +613,11 @@ void link_ELF_aarch64(std::unique_ptr G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); + // Resolve any external section start / end symbols. + Config.PostAllocationPasses.push_back( + createDefineExternalSectionStartAndEndSymbolsPass( + identifyELFSectionStartAndEndSymbols)); + // Add an in-place GOT/TLS/Stubs build pass. Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index a1fe9c5fcd73..52dd83d9040f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -343,24 +343,6 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } -static SectionRangeSymbolDesc -identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { - constexpr StringRef StartSymbolPrefix = "__start"; - constexpr StringRef EndSymbolPrefix = "__end"; - - auto SymName = Sym.getName(); - if (SymName.starts_with(StartSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) - return {*Sec, true}; - } else if (SymName.starts_with(EndSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) - return {*Sec, false}; - } - return {}; -} - void link_ELF_x86_64(std::unique_ptr G, std::unique_ptr Ctx) { PassConfiguration Config; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s new file mode 100644 index 000000000000..f8e7ba96f006 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s @@ -0,0 +1,43 @@ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -position-independent \ +# RUN: -filetype=obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + + .text + .file "elf_section_start_stop.c" + .globl main + .p2align 2 + .type main,@function +main: + adrp x8, z + adrp x9, y + ldr w8, [x8, :lo12:z] + ldr w9, [x9, :lo12:y] + sub w0, w8, w9 + ret +.Lfunc_end0: + .size main, .Lfunc_end0-main + + .type x,@object + .section custom_section,"aw",@progbits + .globl x + .p2align 2 +x: + .word 42 + .size x, 4 + +# jitlink-check: *{8}z = (*{8}y) + 4 + + .type y,@object + .data + .globl y + .p2align 3, 0x0 +y: + .xword __start_custom_section + .size y, 8 + + .type z,@object + .globl z + .p2align 3, 0x0 +z: + .xword __stop_custom_section + .size z, 8 -- GitLab From 9f0321ccf118b37e5cb93cabd2acbf600c36b6ee Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 15:23:50 -0500 Subject: [PATCH 007/343] [Libomptarget] Make plugins depend explicitly on `intrinsics_gen` Summary: It's possible for the OpenMP offloading plugins to be build before tablegen is run despite the fact that we rely on it. Simply make it depend on it currently. --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index b84f3d7b137c..a7350e662a7c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(PluginCommon OBJECT src/RPC.cpp src/Utils/ELF.cpp ) +add_dependencies(PluginCommon intrinsics_gen) # Only enable JIT for those targets that LLVM can support. string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported) -- GitLab From 81e2693c1202d6c4e48dbf2d985153b03cfccb79 Mon Sep 17 00:00:00 2001 From: Hui Date: Sun, 24 Mar 2024 20:52:47 +0000 Subject: [PATCH 008/343] [libc++][test] Fix race condition in condition_variable_any tests (#84788) Some tests in `condition_variable_any` use two `shared_lock` to guard, which does not work. The fix is to make the writer to use `unique_lock` --- .../thread.condition.condvarany/wait_for_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_until_token_pred.pass.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp index 4ea60557d9f8..7a39d1253a33 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(2ms); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp index e96a3e8bd1bc..f322d8cfdc68 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp @@ -63,7 +63,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp index d649db025d75..e7388b9ce0e1 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); -- GitLab From f0a8738401137c32f3e1b9799244ed643feb7bf0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 24 Mar 2024 20:59:53 +0000 Subject: [PATCH 009/343] [VPlan] Generate CalculateTripCountMinusVF for Part 0 only. (NFCI). The value produced by CalculateTripCountMinusVF VPInstructions is independent of the part. Only compute it for part 0 and use that for other parts. --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 + .../AArch64/scalable-strict-fadd.ll | 63 +++---------------- .../AArch64/sve-tail-folding-unroll.ll | 42 ++----------- .../AArch64/uniform-args-call-variants.ll | 13 +--- 4 files changed, 21 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d75e322a74cf..f91997a37993 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -345,6 +345,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name); } case VPInstruction::CalculateTripCountMinusVF: { + if (Part != 0) + return State.get(this, 0, /*IsScalar*/ true); + Value *ScalarTC = State.get(getOperand(0), {0, 0}); Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index fc67fb5aded6..ad6e8534f318 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -403,21 +403,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -492,9 +477,9 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP78:%.*]] = mul i64 [[TMP77]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP79:%.*]] = add i64 [[INDEX]], [[TMP78]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP80:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP81:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP82:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1715,21 +1700,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -1826,9 +1796,9 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -2129,21 +2099,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -2240,9 +2195,9 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index 1a6e83a61ce7..2acc1ddbffea 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -25,21 +25,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -107,9 +92,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 12 ; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP72:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP73:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP74:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -167,21 +152,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -275,9 +245,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP92:%.*]] = mul i64 [[TMP91]], 12 ; CHECK-NEXT: [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP94:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP95:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT14]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP96:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT15]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 4957bbeda671..d8f14f30295b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -40,9 +40,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -71,7 +68,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -129,9 +126,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -160,7 +154,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -207,7 +201,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) -; INTERLEAVE-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1 ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -237,7 +230,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 ; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP0]] ; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: ; INTERLEAVE-NEXT: ret void -- GitLab From 18a49f03aa2b6bfeb073648b9eb75277a2386fc4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2024 14:07:09 -0700 Subject: [PATCH 010/343] [ELF] Merge relaIplt into relaDyn `relaIplt` was added so that IRELATIVE relocations are placed at the end of .rela.dyn (since https://reviews.llvm.org/D65651) or .rela.plt (--pack-dyn-relocs=android[+relr]). Unfortunately, handling `relaIplt` requires special cases all over the code base. We can extend partitionRels/computeRels to partition both RELATIVE and IRELATIVE relocations, rendering `relaIplt` unneeded. The change allows IRELATIVE relocations in the DT_ANDROID_REL[A] table (untested?!), which may be processed before other types of relocations. This seems acceptable for Bionic's DEFINE_IFUNC_FOR use cases. In addition, this change simplies changing .rel[a].dyn to a compact relocation format (CREL). SHF_INFO_LINK is removed from .rel[a].dyn with IRELATIVE relocations. (See https://reviews.llvm.org/D89828). --- lld/ELF/Relocations.cpp | 10 ++--- lld/ELF/SyntheticSections.cpp | 19 +++----- lld/ELF/SyntheticSections.h | 1 - lld/ELF/Writer.cpp | 50 ++++++--------------- lld/test/ELF/aarch64-gnu-ifunc.s | 3 +- lld/test/ELF/arm-gnu-ifunc.s | 3 +- lld/test/ELF/gnu-ifunc-i386.s | 3 +- lld/test/ELF/systemz-ifunc-nonpreemptible.s | 2 +- 8 files changed, 28 insertions(+), 63 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 92a1b9baaca3..33c50133bec4 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1620,12 +1620,8 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // relatively straightforward. We create a PLT entry in Iplt, which is // usually at the end of .plt, which makes an indirect call using a // matching GOT entry in igotPlt, which is usually at the end of .got.plt. - // The GOT entry is relocated using an IRELATIVE relocation in relaIplt, - // which is usually at the end of .rela.plt. Unlike most relocations in - // .rela.plt, which may be evaluated lazily without -z now, dynamic - // loaders evaluate IRELATIVE relocs eagerly, which means that for - // IRELATIVE relocs only, GOT-generating relocations can point directly to - // .got.plt without requiring a separate GOT entry. + // The GOT entry is relocated using an IRELATIVE relocation in relaDyn, + // which is usually at the end of .rela.dyn. // // - Despite the fact that an ifunc does not have a fixed value, compilers // that are not passed -fPIC will assume that they do, and will emit @@ -1665,7 +1661,7 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // section/value fixed. auto *directSym = makeDefined(cast(sym)); directSym->allocateAux(); - addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, + addPltEntry(*in.iplt, *in.igotPlt, *mainPart->relaDyn, target->iRelativeRel, *directSym); sym.allocateAux(); symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 10eda17f0cb3..f924756ddddf 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1267,15 +1267,12 @@ DynamicSection::DynamicSection() // The output section .rela.dyn may include these synthetic sections: // // - part.relaDyn -// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn // - in.relaPlt: this is included if a linker script places .rela.plt inside // .rela.dyn // // DT_RELASZ is the total size of the included sections. static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { size_t size = relaDyn.getSize(); - if (in.relaIplt->getParent() == relaDyn.getParent()) - size += in.relaIplt->getSize(); if (in.relaPlt->getParent() == relaDyn.getParent()) size += in.relaPlt->getSize(); return size; @@ -1372,9 +1369,7 @@ DynamicSection::computeContents() { if (!config->shared && !config->relocatable && !config->zRodynamic) addInt(DT_DEBUG, 0); - if (part.relaDyn->isNeeded() || - (in.relaIplt->isNeeded() && - part.relaDyn->getParent() == in.relaIplt->getParent())) { + if (part.relaDyn->isNeeded()) { addInSec(part.relaDyn->dynamicTag, *part.relaDyn); entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(*part.relaDyn)); @@ -1657,10 +1652,6 @@ void RelocationBaseSection::finalizeContents() { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.gotPlt->getParent()->sectionIndex; } - if (in.relaIplt.get() == this && in.igotPlt->getParent()) { - getParent()->flags |= ELF::SHF_INFO_LINK; - getParent()->info = in.igotPlt->getParent()->sectionIndex; - } } void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) { @@ -1674,6 +1665,11 @@ void RelocationBaseSection::computeRels() { SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); parallelForEach(relocs, [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); }); + + auto irelative = std::partition( + relocs.begin() + numRelativeRelocs, relocs.end(), + [t = target->iRelativeRel](auto &r) { return r.type != t; }); + // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset // is to make results easier to read. @@ -1682,7 +1678,7 @@ void RelocationBaseSection::computeRels() { parallelSort(relocs.begin(), nonRelative, [&](auto &a, auto &b) { return a.r_offset < b.r_offset; }); // Non-relative relocations are few, so don't bother with parallelSort. - llvm::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) { + llvm::sort(nonRelative, irelative, [&](auto &a, auto &b) { return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset); }); } @@ -3843,7 +3839,6 @@ void InStruct::reset() { ppc32Got2.reset(); ibtPlt.reset(); relaPlt.reset(); - relaIplt.reset(); shStrTab.reset(); strTab.reset(); symTab.reset(); diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index b41e69410054..fa21b80a5a5e 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -1358,7 +1358,6 @@ struct InStruct { std::unique_ptr ppc32Got2; std::unique_ptr ibtPlt; std::unique_ptr relaPlt; - std::unique_ptr relaIplt; std::unique_ptr shStrTab; std::unique_ptr strTab; std::unique_ptr symTab; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 4eca7b22e90b..40d617b7fdf3 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -449,8 +449,8 @@ template void elf::createSyntheticSections() { add(*part.dynamic); add(*part.dynStrTab); - add(*part.relaDyn); } + add(*part.relaDyn); if (config->relrPackDynRelocs) { part.relrDyn = std::make_unique>(threadCount); @@ -550,17 +550,6 @@ template void elf::createSyntheticSections() { /*threadCount=*/1); add(*in.relaPlt); - // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative - // relocations are processed last by the dynamic loader. We cannot place the - // iplt section in .rel.dyn when Android relocation packing is enabled because - // that would cause a section type mismatch. However, because the Android - // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired - // behaviour by placing the iplt section in .rel.plt. - in.relaIplt = std::make_unique>( - config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, - /*sort=*/false, /*threadCount=*/1); - add(*in.relaIplt); - if ((config->emachine == EM_386 || config->emachine == EM_X86_64) && (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) { in.ibtPlt = std::make_unique(); @@ -1071,20 +1060,18 @@ void PhdrEntry::add(OutputSection *sec) { sec->ptLoad = this; } -// The beginning and the ending of .rel[a].plt section are marked -// with __rel[a]_iplt_{start,end} symbols if it is a statically linked -// executable. The runtime needs these symbols in order to resolve -// all IRELATIVE relocs on startup. For dynamic executables, we don't -// need these symbols, since IRELATIVE relocs are resolved through GOT -// and PLT. For details, see http://www.airs.com/blog/archives/403. +// A statically linked position-dependent executable should only contain +// IRELATIVE relocations and no other dynamic relocations. Encapsulation symbols +// __rel[a]_iplt_{start,end} will be defined for .rel[a].dyn, to be +// processed by the libc runtime. Other executables or DSOs use dynamic tags +// instead. template void Writer::addRelIpltSymbols() { if (config->isPic) return; - // By default, __rela_iplt_{start,end} belong to a dummy section 0 - // because .rela.plt might be empty and thus removed from output. - // We'll override Out::elfHeader with In.relaIplt later when we are - // sure that .rela.plt exists in output. + // __rela_iplt_{start,end} are initially defined relative to dummy section 0. + // We'll override Out::elfHeader with relaDyn later when we are sure that + // .rela.dyn will be present in the output. ElfSym::relaIpltStart = addOptionalRegular( config->isRela ? "__rela_iplt_start" : "__rel_iplt_start", Out::elfHeader, 0, STV_HIDDEN); @@ -1110,11 +1097,11 @@ template void Writer::setReservedSymbolSections() { ElfSym::globalOffsetTable->section = sec; } - // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. - if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { - ElfSym::relaIpltStart->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->value = in.relaIplt->getSize(); + // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn. + if (ElfSym::relaIpltStart && mainPart->relaDyn->isNeeded()) { + ElfSym::relaIpltStart->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->value = mainPart->relaDyn->getSize(); } PhdrEntry *last = nullptr; @@ -1470,14 +1457,6 @@ static void sortSection(OutputSection &osec, if (name == ".init" || name == ".fini") return; - // IRelative relocations that usually live in the .rel[a].dyn section should - // be processed last by the dynamic loader. To achieve that we add synthetic - // sections in the required order from the beginning so that the in.relaIplt - // section is placed last in an output section. Here we just do not apply - // sorting for an output section which holds the in.relaIplt section. - if (in.relaIplt->getParent() == &osec) - return; - // Sort input sections by priority using the list provided by // --symbol-ordering-file or --shuffle-sections=. This is a least significant // digit radix sort. The sections may be sorted stably again by a more @@ -2196,7 +2175,6 @@ template void Writer::finalizeSections() { finalizeSynthetic(in.mipsGot.get()); finalizeSynthetic(in.igotPlt.get()); finalizeSynthetic(in.gotPlt.get()); - finalizeSynthetic(in.relaIplt.get()); finalizeSynthetic(in.relaPlt.get()); finalizeSynthetic(in.plt.get()); finalizeSynthetic(in.iplt.get()); diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s index dee24779d913..d76b54eabf8a 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc.s +++ b/lld/test/ELF/aarch64-gnu-ifunc.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_RELA // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0x158 // CHECK-NEXT: Size: 48 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 8 // CHECK-NEXT: EntrySize: 24 // CHECK-NEXT: } diff --git a/lld/test/ELF/arm-gnu-ifunc.s b/lld/test/ELF/arm-gnu-ifunc.s index 562478256fd8..d49ca18e991e 100644 --- a/lld/test/ELF/arm-gnu-ifunc.s +++ b/lld/test/ELF/arm-gnu-ifunc.s @@ -30,13 +30,12 @@ _start: // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: 0x100F4 // CHECK-NEXT: Offset: 0xF4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK: Name: .iplt // CHECK-NEXT: Type: SHT_PROGBITS // CHECK-NEXT: Flags [ diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s index b502fd6e9ae2..43b19b27ea4e 100644 --- a/lld/test/ELF/gnu-ifunc-i386.s +++ b/lld/test/ELF/gnu-ifunc-i386.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0xD4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 4 // CHECK-NEXT: EntrySize: 8 // CHECK-NEXT: } diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s index 5056db302ca1..892bbde8d9c7 100644 --- a/lld/test/ELF/systemz-ifunc-nonpreemptible.s +++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s @@ -10,7 +10,7 @@ # CHECK: Section Headers: # CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al # CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 -# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 AI 0 4 8 +# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 A 0 0 8 # CHECK-NEXT: [ 2] .text PROGBITS 0000000001001188 000188 00001c 00 AX 0 0 4 # CHECK-NEXT: [ 3] .iplt PROGBITS 00000000010011b0 0001b0 000040 00 AX 0 0 16 # CHECK-NEXT: [ 4] .got.plt PROGBITS 00000000010021f0 0001f0 000010 00 WA 0 0 8 -- GitLab From 67f2267ae006f35edfdf36567646403e61527d1b Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Sun, 24 Mar 2024 14:24:44 -0700 Subject: [PATCH 011/343] [M68k][NFC] Suppress warning on an unused variable when assertion is disabled NFC. --- llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp index 158393f02a24..786aa7bcb64e 100644 --- a/llvm/lib/Target/M68k/M68kISelLowering.cpp +++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp @@ -939,6 +939,7 @@ SDValue M68kTargetLowering::LowerFormalArguments( for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; assert(VA.getValNo() != LastVal && "Same value in different locations"); + (void)LastVal; LastVal = VA.getValNo(); -- GitLab From cceedc939a43c7c732a5888364251775bffc2dba Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sun, 24 Mar 2024 15:22:40 -0700 Subject: [PATCH 012/343] [clang-format] Fix a crash with AlignArrayOfStructures option (#86420) Fixes #86109. --- clang/lib/Format/WhitespaceManager.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index fef85abf79a3..710bf8d8a8ec 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -1491,7 +1491,7 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start, : Cell); // Go to the next non-comment and ensure there is a break in front const auto *NextNonComment = C.Tok->getNextNonComment(); - while (NextNonComment->is(tok::comma)) + while (NextNonComment && NextNonComment->is(tok::comma)) NextNonComment = NextNonComment->getNextNonComment(); auto j = i; while (j < End && Changes[j].Tok != NextNonComment) diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index cf8d6ab691d9..03005384a6f6 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -21100,7 +21100,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresRightAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", " @@ -21353,7 +21360,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresLeftAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", {\"x\", " -- GitLab From 230b1895c493c511c11541af3b5bc819887c82a8 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Sun, 24 Mar 2024 18:56:47 -0700 Subject: [PATCH 013/343] [mlir][spirv] Add folding for [S|U|LessThan[Equal] (#85435) Add missing constant propogation folder for [S|U]LessThan[Equal]. Implement additional folding when the operands are equal for all ops. Allows for constant folding in the IndexToSPIRV pass. Part of work #70704 --- .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td | 8 + .../SPIRV/IR/SPIRVCanonicalization.cpp | 82 ++++++++ .../SPIRV/Transforms/canonicalize.mlir | 176 ++++++++++++++++++ 3 files changed, 266 insertions(+) diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td index 3ee239d6e1e3..14d639bc26f2 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td @@ -716,6 +716,8 @@ def SPIRV_SLessThanOp : SPIRV_LogicalBinaryOp<"SLessThan", ``` }]; + + let hasFolder = 1; } // ----- @@ -745,6 +747,8 @@ def SPIRV_SLessThanEqualOp : SPIRV_LogicalBinaryOp<"SLessThanEqual", %5 = spirv.SLessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -886,6 +890,8 @@ def SPIRV_ULessThanOp : SPIRV_LogicalBinaryOp<"ULessThan", %5 = spirv.ULessThan %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -949,6 +955,8 @@ def SPIRV_ULessThanEqualOp : SPIRV_LogicalBinaryOp<"ULessThanEqual", %5 = spirv.ULessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } #endif // MLIR_DIALECT_SPIRV_IR_LOGICAL_OPS diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index ff4bace9a4d8..3f1a7826f8b6 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -880,6 +880,88 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) { }); } +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::SLessThanOp::fold(spirv::SLessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.slt(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::SLessThanEqualOp::fold(spirv::SLessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.sle(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ULessThanOp::fold(spirv::ULessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ult(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::ULessThanEqualOp::fold(spirv::ULessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ule(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + //===----------------------------------------------------------------------===// // spirv.ShiftLeftLogical //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index de21d114e9fc..ed0bd070c43c 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> { // ----- +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @slt_same +func.func @slt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.SLessThan %arg0, %arg0 : i32 + %1 = spirv.SLessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_slt +func.func @const_fold_scalar_slt() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThan %c5, %c6 : i32 + %1 = spirv.SLessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_slt +func.func @const_fold_vector_slt() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.SLessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @sle_same +func.func @sle_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.SLessThanEqual %arg0, %arg0 : i32 + %1 = spirv.SLessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_sle +func.func @const_fold_scalar_sle() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThanEqual %c5, %c6 : i32 + %1 = spirv.SLessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_sle +func.func @const_fold_vector_sle() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.SLessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ult_same +func.func @ult_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.ULessThan %arg0, %arg0 : i32 + %1 = spirv.ULessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ult +func.func @const_fold_scalar_ult() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThan %c5, %cn6 : i32 + %1 = spirv.ULessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ult +func.func @const_fold_vector_ult() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.ULessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ule_same +func.func @ule_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.ULessThanEqual %arg0, %arg0 : i32 + %1 = spirv.ULessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ule +func.func @const_fold_scalar_ule() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThanEqual %c5, %cn6 : i32 + %1 = spirv.ULessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ule +func.func @const_fold_vector_ule() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.ULessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.LeftShiftLogical //===----------------------------------------------------------------------===// -- GitLab From 7d2d8e2a7245e4e64da22cb3c422ea3be5a0bf0a Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Mon, 25 Mar 2024 10:59:42 +0900 Subject: [PATCH 014/343] [mlir][complex] Fastmath flag for the trigonometric ops in complex (#85563) Support Fastmath flag to convert trigonometric ops in the complex dialect. See: https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981 --- .../ComplexToStandard/ComplexToStandard.cpp | 50 +++++++++++-------- .../convert-to-standard.mlir | 46 +++++++++++++++++ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp index 76729278ec1b..17f64f1b65b7 100644 --- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp +++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp @@ -196,6 +196,7 @@ struct TrigonometricOpConversion : public OpConversionPattern { auto loc = op.getLoc(); auto type = cast(adaptor.getComplex().getType()); auto elementType = cast(type.getElementType()); + arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); Value real = rewriter.create(loc, elementType, adaptor.getComplex()); @@ -207,14 +208,14 @@ struct TrigonometricOpConversion : public OpConversionPattern { // implementation in the subclass to combine them. Value half = rewriter.create( loc, elementType, rewriter.getFloatAttr(elementType, 0.5)); - Value exp = rewriter.create(loc, imag); - Value scaledExp = rewriter.create(loc, half, exp); - Value reciprocalExp = rewriter.create(loc, half, exp); - Value sin = rewriter.create(loc, real); - Value cos = rewriter.create(loc, real); + Value exp = rewriter.create(loc, imag, fmf); + Value scaledExp = rewriter.create(loc, half, exp, fmf); + Value reciprocalExp = rewriter.create(loc, half, exp, fmf); + Value sin = rewriter.create(loc, real, fmf); + Value cos = rewriter.create(loc, real, fmf); auto resultPair = - combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter); + combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf); rewriter.replaceOpWithNewOp(op, type, resultPair.first, resultPair.second); @@ -223,15 +224,17 @@ struct TrigonometricOpConversion : public OpConversionPattern { virtual std::pair combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const = 0; + Value cos, ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const = 0; }; struct CosOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex cosine is defined as; // cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy))) // Plugging in: @@ -241,10 +244,12 @@ struct CosOpConversion : public TrigonometricOpConversion { // We get: // Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x // Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x - Value sum = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultReal = rewriter.create(loc, sum, cos); - Value diff = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultImag = rewriter.create(loc, diff, sin); + Value sum = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultReal = rewriter.create(loc, sum, cos, fmf); + Value diff = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultImag = rewriter.create(loc, diff, sin, fmf); return {resultReal, resultImag}; } }; @@ -813,9 +818,10 @@ struct NegOpConversion : public OpConversionPattern { struct SinOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex sine is defined as; // sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy))) // Plugging in: @@ -825,10 +831,12 @@ struct SinOpConversion : public TrigonometricOpConversion { // We get: // Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x // Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x - Value sum = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultReal = rewriter.create(loc, sum, sin); - Value diff = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultImag = rewriter.create(loc, diff, cos); + Value sum = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultReal = rewriter.create(loc, sum, sin, fmf); + Value diff = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultImag = rewriter.create(loc, diff, cos, fmf); return {resultReal, resultImag}; } }; diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir index 5918ff2e0f36..bac94aae6b74 100644 --- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir +++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir @@ -1834,3 +1834,49 @@ func.func @complex_sqrt_with_fmf(%arg: complex) -> complex { // CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32 // CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex // CHECK: return %[[VAR41]] : complex + +// ----- + +// CHECK-LABEL: func @complex_cos_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_cos_with_fmf(%arg: complex) -> complex { + %cos = complex.cos %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] + +// ----- + +// CHECK-LABEL: func @complex_sin_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_sin_with_fmf(%arg: complex) -> complex { + %cos = complex.sin %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] -- GitLab From 2e4e04c59043a645572cf548de5d9c333a5d6641 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Mon, 25 Mar 2024 10:06:12 +0800 Subject: [PATCH 015/343] [X86][BF16] Do not lower to VCVTNEPS2BF16 without AVX512VL (#86395) Fixes: #86305 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ++- llvm/test/CodeGen/X86/pr86305.ll | 74 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/X86/pr86305.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 35f756ea5e1d..9acbe17d0bca 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21512,7 +21512,9 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { } if (VT.getScalarType() == MVT::bf16) { - if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT)) + if (SVT.getScalarType() == MVT::f32 && + ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) return Op; return SDValue(); } @@ -21619,7 +21621,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op, SDLoc DL(Op); MVT SVT = Op.getOperand(0).getSimpleValueType(); - if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) { + if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) { SDValue Res; Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0)); Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res); diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll new file mode 100644 index 000000000000..79b42bb2532c --- /dev/null +++ b/llvm/test/CodeGen/X86/pr86305.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16 | FileCheck %s + +define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { +; CHECK-LABEL: add: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movq %rdx, %rbx +; CHECK-NEXT: movzwl (%rsi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm1 +; CHECK-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, (%rbx) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq + %a = load bfloat, ptr %pa + %b = load bfloat, ptr %pb + %add = fadd bfloat %a, %b + store bfloat %add, ptr %pc + ret void +} + +define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind { +; CHECK-LABEL: fptrunc_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %ebx +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %ebp +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r14d +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r15d +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %eax +; CHECK-NEXT: vmovd %r15d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r14d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq + %b = fptrunc <4 x float> %a to <4 x bfloat> + ret <4 x bfloat> %b +} -- GitLab From 5e5b6561029665e69e033cff4216fecb78302259 Mon Sep 17 00:00:00 2001 From: Sergei Barannikov Date: Mon, 25 Mar 2024 05:13:48 +0300 Subject: [PATCH 016/343] [MC] Make `MCParsedAsmOperand::getReg()` return `MCRegister` (#86444) --- .../llvm/MC/MCParser/MCParsedAsmOperand.h | 3 ++- .../llvm/MC/MCParser/MCTargetAsmParser.h | 4 +-- llvm/lib/MC/MCParser/MCTargetAsmParser.cpp | 6 +++++ .../AArch64/AsmParser/AArch64AsmParser.cpp | 2 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2 +- .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 2 +- .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 2 +- .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 2 +- .../Target/CSKY/AsmParser/CSKYAsmParser.cpp | 2 +- .../Hexagon/AsmParser/HexagonAsmParser.cpp | 2 +- .../Target/Lanai/AsmParser/LanaiAsmParser.cpp | 2 +- .../AsmParser/LoongArchAsmParser.cpp | 4 +-- .../Target/M68k/AsmParser/M68kAsmParser.cpp | 4 +-- .../MSP430/AsmParser/MSP430AsmParser.cpp | 2 +- .../Target/Mips/AsmParser/MipsAsmParser.cpp | 2 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 26 ++++++++++--------- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 +-- .../Target/Sparc/AsmParser/SparcAsmParser.cpp | 2 +- .../SystemZ/AsmParser/SystemZAsmParser.cpp | 2 +- llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp | 2 +- .../AsmParser/WebAssemblyAsmParser.cpp | 2 +- llvm/lib/Target/X86/AsmParser/X86Operand.h | 2 +- .../Xtensa/AsmParser/XtensaAsmParser.cpp | 2 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 2 +- 24 files changed, 46 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 0c9668904e82..27ecb7b85d22 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -15,6 +15,7 @@ namespace llvm { +class MCRegister; class raw_ostream; /// MCParsedAsmOperand - This abstract class represents a source-level assembly @@ -57,7 +58,7 @@ public: virtual bool isImm() const = 0; /// isReg - Is this a register operand? virtual bool isReg() const = 0; - virtual unsigned getReg() const = 0; + virtual MCRegister getReg() const = 0; /// isMem - Is this a memory operand? virtual bool isMem() const = 0; diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 7edd3f8ce490..49ce417e6fbb 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -514,9 +514,7 @@ public: /// by the tied-operands checks in the AsmMatcher. This method can be /// overridden to allow e.g. a sub- or super-register as the tied operand. virtual bool areEqualRegs(const MCParsedAsmOperand &Op1, - const MCParsedAsmOperand &Op2) const { - return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); - } + const MCParsedAsmOperand &Op2) const; // Return whether this parser uses assignment statements with equals tokens virtual bool equalIsAsmAssignment() { return true; }; diff --git a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp index 0db5fb36f795..665d92eb9a21 100644 --- a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp +++ b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp @@ -8,6 +8,7 @@ #include "llvm/MC/MCParser/MCTargetAsmParser.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCRegister.h" using namespace llvm; @@ -48,3 +49,8 @@ ParseStatus MCTargetAsmParser::parseDirective(AsmToken DirectiveID) { return ParseStatus::Failure; return ParseStatus::NoMatch; } + +bool MCTargetAsmParser::areEqualRegs(const MCParsedAsmOperand &Op1, + const MCParsedAsmOperand &Op2) const { + return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); +} diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index b807aaf76fdb..21643ebb4138 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -654,7 +654,7 @@ public: return Barrier.HasnXSModifier; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..4648df199c74 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -981,7 +981,7 @@ public: return Imm.Type; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isRegKind()); return Reg.RegNo; } diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 9cfdb15a0f43..2ad576ab1a9c 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -1002,7 +1002,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp index db4aa03437c6..383dfcc31117 100644 --- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp +++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp @@ -195,7 +195,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_Memri) && "Invalid access!"); return RegImm.Reg; diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 1688355f427c..9672ed009e9b 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -148,7 +148,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp index 4711e58bbed6..30bd3dcefa60 100644 --- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp +++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp @@ -400,7 +400,7 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index 864591d4eb95..092cccbcca9c 100644 --- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -245,7 +245,7 @@ public: /// getEndLoc - Get the location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp index ff3649b77e35..6ab1375b974e 100644 --- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp +++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp @@ -151,7 +151,7 @@ public: // getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isReg() && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp index cf163e4e1200..20284b18428b 100644 --- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp +++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp @@ -467,9 +467,9 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } const MCExpr *getImm() const { diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp index b2c0fda1ccc2..126176133dc0 100644 --- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp +++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp @@ -157,7 +157,7 @@ public: bool isDReg() const; bool isFPDReg() const; bool isFPCReg() const; - unsigned getReg() const override; + MCRegister getReg() const override; void addRegOperands(MCInst &Inst, unsigned N) const; static std::unique_ptr createMemOp(M68kMemOp MemOp, SMLoc Start, @@ -312,7 +312,7 @@ bool M68kOperand::isReg() const { return Kind == KindTy::MemOp && MemOp.Op == M68kMemOp::Kind::Reg; } -unsigned M68kOperand::getReg() const { +MCRegister M68kOperand::getReg() const { assert(isReg()); return MemOp.OuterReg; } diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp index 818a468612a5..2bc1a89ef59c 100644 --- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp +++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp @@ -183,7 +183,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Reg && "Invalid access!"); return Reg; } diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 9d6e8dc573a8..076e0a20cb97 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -1458,7 +1458,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { // As a special case until we sort out the definition of div/divu, accept // $0/$zero here so that MCK_ZERO works correctly. if (Kind == k_RegisterIndex && RegIdx.Index == 0 && diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index 8108cfa521c8..55978af38000 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -276,9 +276,11 @@ public: return TLSReg.Sym; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Not implemented"); } + + unsigned getRegNum() const { assert(isRegNumber() && "Invalid access!"); - return (unsigned) Imm.Val; + return (unsigned)Imm.Val; } unsigned getFpReg() const { @@ -459,22 +461,22 @@ public: void addRegGPRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegsNoR0[getRegNum()])); } void addRegG8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegs[getRegNum()])); } void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegsNoX0[getRegNum()])); } void addRegG8pRCOperands(MCInst &Inst, unsigned N) const { @@ -498,12 +500,12 @@ public: void addRegF4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegF8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegFpRCOperands(MCInst &Inst, unsigned N) const { @@ -513,12 +515,12 @@ public: void addRegVFRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VFRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VFRegs[getRegNum()])); } void addRegVRRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VRegs[getRegNum()])); } void addRegVSRCOperands(MCInst &Inst, unsigned N) const { @@ -538,12 +540,12 @@ public: void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegSPERCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(SPERegs[getReg()])); + Inst.addOperand(MCOperand::createReg(SPERegs[getRegNum()])); } void addRegACCRCOperands(MCInst &Inst, unsigned N) const { diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index cb2ba52390e2..5e594d6cad6b 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -977,9 +977,9 @@ public: return Imm.IsRV64; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } StringRef getSysReg() const { diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp index be4ec1e9dce2..67e2b9d7c997 100644 --- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp +++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp @@ -307,7 +307,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp index a58e8e0dfedf..f2c04215d12d 100644 --- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp +++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp @@ -227,7 +227,7 @@ public: bool isReg(RegisterKind RegKind) const { return Kind == KindReg && Reg.Kind == RegKind; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindReg && "Not a register"); return Reg.Num; } diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp index f9e30a3a9378..691fe8fe3aa4 100644 --- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp +++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp @@ -344,7 +344,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 3cc4d50271eb..020c0d6229d2 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -100,7 +100,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand { bool isReg() const override { return false; } bool isBrList() const { return Kind == BrList; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Assembly inspects a register operand"); return 0; } diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h index 641158cb351f..78669784dd03 100644 --- a/llvm/lib/Target/X86/AsmParser/X86Operand.h +++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h @@ -167,7 +167,7 @@ struct X86Operand final : public MCParsedAsmOperand { Tok.Length = Value.size(); } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNo; } diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp index 3f808298527f..1fa00af2111e 100644 --- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp +++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp @@ -244,7 +244,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 5df7990d8fc2..0a2b1cf0f9a6 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -2519,7 +2519,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) { // Check for register operands, including sub-classes. OS << " if (Operand.isReg()) {\n"; OS << " MatchClassKind OpKind;\n"; - OS << " switch (Operand.getReg()) {\n"; + OS << " switch (Operand.getReg().id()) {\n"; OS << " default: OpKind = InvalidMatchClass; break;\n"; for (const auto &RC : Info.RegisterClasses) OS << " case " << RC.first->getValueAsString("Namespace") -- GitLab From 6af6416e89de1f4656a145c4843226b468718434 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 11:44:16 +0800 Subject: [PATCH 017/343] [RISCV] Add a tune feature to disable stripping W suffix (#86255) We have a hidden option to disable it, but I'd like to make it a tune feature. For some implementations, instructions with W suffix would be less costly as they only perform on 32 bits data. Though we may lose some chances to compress. --- llvm/lib/Target/RISCV/RISCVFeatures.td | 4 ++ llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 2 +- llvm/test/CodeGen/RISCV/strip-w-suffix.ll | 74 +++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index f3e641e25018..6ef2289bb4be 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1226,6 +1226,10 @@ def TuneNoSinkSplatOperands "false", "Disable sink splat operands to enable .vx, .vf," ".wx, and .wf instructions">; +def TuneNoStripWSuffix + : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false", + "Disable strip W suffix">; + def TuneConditionalCompressedMoveFusion : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion", "true", "Enable branch+c.mv fusion">; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index dcf70e8cad64..39d420c2fbf0 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -672,7 +672,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII, const RISCVSubtarget &ST, MachineRegisterInfo &MRI) { - if (DisableStripWSuffix) + if (DisableStripWSuffix || !ST.enableStripWSuffix()) return false; bool MadeChange = false; diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll new file mode 100644 index 000000000000..4124b3d0d360 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=STRIP %s +; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=NO-STRIP %s + +define i32 @addiw(i32 %a) { +; STRIP-LABEL: addiw: +; STRIP: # %bb.0: +; STRIP-NEXT: lui a1, 1 +; STRIP-NEXT: addi a1, a1, -1 +; STRIP-NEXT: addw a0, a0, a1 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addiw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: lui a1, 1 +; NO-STRIP-NEXT: addiw a1, a1, -1 +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: ret + %ret = add i32 %a, 4095 + ret i32 %ret +} + +define i32 @addw(i32 %a, i32 %b) { +; STRIP-LABEL: addw: +; STRIP: # %bb.0: +; STRIP-NEXT: add a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %add = add i32 %a, %b + %ret = add i32 %add, 1024 + ret i32 %ret +} + +define i32 @mulw(i32 %a, i32 %b) { +; STRIP-LABEL: mulw: +; STRIP: # %bb.0: +; STRIP-NEXT: mul a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: mulw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: mulw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %mul = mul i32 %a, %b + %ret = add i32 %mul, 1024 + ret i32 %ret +} + +define i32 @slliw(i32 %a) { +; STRIP-LABEL: slliw: +; STRIP: # %bb.0: +; STRIP-NEXT: slli a0, a0, 1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: slliw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: slliw a0, a0, 1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %shl = shl i32 %a, 1 + %ret = add i32 %shl, 1024 + ret i32 %ret +} -- GitLab From d9746a6a5d523e21eee2c1b50c2f08aa19396965 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 12:42:59 +0800 Subject: [PATCH 018/343] [RISCV][NFC] Pass LMUL to copyPhysRegVector The opcode will be determined by LMUL. Reviewers: preames, lukel97, topperc Reviewed By: lukel97, topperc Pull Request: https://github.com/llvm/llvm-project/pull/84448 --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 52 ++++++++++++------------ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 4ca300f9151e..14c2d41e80f1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -299,36 +299,36 @@ void RISCVInstrInfo::copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF) const { + RISCVII::VLMUL LMul, unsigned NF) const { const TargetRegisterInfo *TRI = STI.getRegisterInfo(); - RISCVII::VLMUL LMul; + unsigned Opc; unsigned SubRegIdx; unsigned VVOpc, VIOpc; - switch (Opc) { + switch (LMul) { default: llvm_unreachable("Impossible LMUL for vector register copy."); - case RISCV::VMV1R_V: - LMul = RISCVII::LMUL_1; + case RISCVII::LMUL_1: + Opc = RISCV::VMV1R_V; SubRegIdx = RISCV::sub_vrm1_0; VVOpc = RISCV::PseudoVMV_V_V_M1; VIOpc = RISCV::PseudoVMV_V_I_M1; break; - case RISCV::VMV2R_V: - LMul = RISCVII::LMUL_2; + case RISCVII::LMUL_2: + Opc = RISCV::VMV2R_V; SubRegIdx = RISCV::sub_vrm2_0; VVOpc = RISCV::PseudoVMV_V_V_M2; VIOpc = RISCV::PseudoVMV_V_I_M2; break; - case RISCV::VMV4R_V: - LMul = RISCVII::LMUL_4; + case RISCVII::LMUL_4: + Opc = RISCV::VMV4R_V; SubRegIdx = RISCV::sub_vrm4_0; VVOpc = RISCV::PseudoVMV_V_V_M4; VIOpc = RISCV::PseudoVMV_V_I_M4; break; - case RISCV::VMV8R_V: + case RISCVII::LMUL_8: assert(NF == 1); - LMul = RISCVII::LMUL_8; + Opc = RISCV::VMV8R_V; SubRegIdx = RISCV::sub_vrm1_0; // There is no sub_vrm8_0. VVOpc = RISCV::PseudoVMV_V_V_M8; VIOpc = RISCV::PseudoVMV_V_I_M8; @@ -505,87 +505,87 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // VR->VR copies. if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1); return; } if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2); return; } if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4); return; } if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV8R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_8); return; } if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/2); return; } if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/2); return; } if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4, /*NF=*/2); return; } if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/3); return; } if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/3); return; } if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/4); return; } if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/4); return; } if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/5); return; } if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/6); return; } if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/7); return; } if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/8); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 8a312ee5e779..dd049fca0597 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -69,7 +69,7 @@ public: void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF = 1) const; + RISCVII::VLMUL LMul, unsigned NF = 1) const; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override; -- GitLab From 373e77b4c0ad9b0bf370f0e5a32a4100a5459d82 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 25 Mar 2024 13:08:56 +0800 Subject: [PATCH 019/343] [RISCV] Generalize (sub zext, zext) -> (sext (sub zext, zext)) to add (#86248) This generalizes the combine added in #82455 to other binary ops, beginning with adds in this patch. Because the two zext operands are always +ve when treated as signed, and we don't get any overflow since the add is carried out in at least N * 2 bits of the narrow type, the result of the add will always be +ve. So we can use a zext for the outer extend, unlike sub which may produce a -ve result from two +ve operands. Although we could still use sext for add, I plan to add support for other binary ops like mul in a later patch, but mul requires zext to be correct (because the maximum value will take up the full N * 2 bits). So I've opted to use zext here too for consistency. Alive2 proof: https://alive2.llvm.org/ce/z/PRNsUM --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 75 +++++++--- .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll | 32 ++--- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 128 ++++++++---------- 3 files changed, 125 insertions(+), 110 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5214595485ca..4ace50aa477a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12913,6 +12913,55 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); } +// add (zext, zext) -> zext (add (zext, zext)) +// sub (zext, zext) -> sext (sub (zext, zext)) +// +// where the sum of the extend widths match, and the the range of the bin op +// fits inside the width of the narrower bin op. (For profitability on rvv, we +// use a power of two for both inner and outer extend.) +// +// TODO: Extend this to other binary ops +static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) { + + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND) + return SDValue(); + if (!N0.hasOneUse() || !N1.hasOneUse()) + return SDValue(); + + SDValue Src0 = N0.getOperand(0); + SDValue Src1 = N1.getOperand(0); + EVT SrcVT = Src0.getValueType(); + if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) || + SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 || + SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); + EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); + + Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); + Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); + + // Src0 and Src1 are zero extended, so they're always positive if signed. + // + // sub can produce a negative from two positive operands, so it needs sign + // extended. Other nodes produce a positive from two positive operands, so + // zero extend instead. + unsigned OuterExtend = + N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + + return DAG.getNode( + OuterExtend, SDLoc(N), VT, + DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1)); +} + // Try to turn (add (xor bool, 1) -1) into (neg bool). static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); @@ -12950,6 +12999,8 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, return V; if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) return V; + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (add (select lhs, rhs, cc, 0, y), x) -> // (select lhs, rhs, cc, x, (add x, y)) @@ -13017,28 +13068,8 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, } } - // sub (zext, zext) -> sext (sub (zext, zext)) - // where the sum of the extend widths match, and the inner zexts - // add at least one bit. (For profitability on rvv, we use a - // power of two for both inner and outer extend.) - if (VT.isVector() && Subtarget.getTargetLowering()->isTypeLegal(VT) && - N0.getOpcode() == N1.getOpcode() && N0.getOpcode() == ISD::ZERO_EXTEND && - N0.hasOneUse() && N1.hasOneUse()) { - SDValue Src0 = N0.getOperand(0); - SDValue Src1 = N1.getOperand(0); - EVT SrcVT = Src0.getValueType(); - if (Subtarget.getTargetLowering()->isTypeLegal(SrcVT) && - SrcVT == Src1.getValueType() && SrcVT.getScalarSizeInBits() >= 8 && - SrcVT.getScalarSizeInBits() < VT.getScalarSizeInBits() / 2) { - LLVMContext &C = *DAG.getContext(); - EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); - EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); - Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); - Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); - return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, - DAG.getNode(ISD::SUB, SDLoc(N), NarrowVT, Src0, Src1)); - } - } + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (sub x, (select lhs, rhs, cc, 0, y)) -> // (select lhs, rhs, cc, x, (sub x, y)) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll index 57a72c639b33..bc0bf5dd76ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll @@ -385,12 +385,12 @@ define <32 x i64> @vwaddu_v32i64(ptr %x, ptr %y) nounwind { define <2 x i32> @vwaddu_v2i32_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -912,12 +912,12 @@ define <4 x i64> @crash(<4 x i16> %x, <4 x i16> %y) { define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -930,12 +930,12 @@ define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -948,12 +948,12 @@ define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i16(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: vle16.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i16>, ptr %x %b = load <2 x i16>, ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 66a7eea18be5..0a7051633a19 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -468,11 +468,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -555,10 +553,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -588,11 +586,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -675,10 +671,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -708,11 +704,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -795,10 +789,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -828,11 +822,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -915,10 +907,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -948,11 +940,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1035,10 +1025,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1068,11 +1058,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1155,10 +1143,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1188,11 +1176,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1275,10 +1261,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1308,11 +1294,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From babbdad15b8049a6a78087d15a163d897f07d320 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:23:40 +0100 Subject: [PATCH 020/343] [AMDGPU] Handle non-register operands for S_SUB/ADD_U64_PSEUDO (#86104) This pseudo uses SSrc_b64 so it allows both an immediate or a register, but the lowering crashed on immediate operands. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- .../CodeGen/AMDGPU/add_sub_u64_pseudos.mir | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 7f0cff72c186..d437f339a687 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4859,8 +4859,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( if (Subtarget->hasScalarAddSub64()) { unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) - .addReg(Src0.getReg()) - .addReg(Src1.getReg()); + .add(Src0) + .add(Src1); } else { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const TargetRegisterClass *BoolRC = TRI->getBoolRC(); diff --git a/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir new file mode 100644 index 000000000000..cba114c3568a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir @@ -0,0 +1,68 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX11 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX12 %s + +--- +name: reg_ops +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: reg_ops + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub0 + ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY2]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], [[COPY3]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: reg_ops + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], [[DEF1]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = IMPLICIT_DEF + %2:sreg_64 = S_ADD_U64_PSEUDO %0, %1, implicit-def $scc +... + +--- +name: lhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: lhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 6565, [[COPY]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 0, [[COPY1]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: lhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 6565, [[DEF]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO 6565, %0, implicit-def $scc +... + +--- +name: rhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: rhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 6565, implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], 0, implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: rhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], 6565 + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO %0, 6565, implicit-def $scc +... -- GitLab From fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:40:35 +0100 Subject: [PATCH 021/343] [RFC][TableGen] Restructure TableGen Source (#80847) Refactor of the llvm-tblgen source into: - a "Basic" library, which contains the bare minimum utilities to build `llvm-min-tablegen` - a "Common" library which contains all of the helpers for TableGen backends. Such helpers can be shared by more than one backend, and even unit tested (e.g. CodeExpander is, maybe we can add more over time) Fixes #80647 --- llvm/unittests/TableGen/CMakeLists.txt | 2 +- llvm/unittests/TableGen/CodeExpanderTest.cpp | 4 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/AsmWriterEmitter.cpp | 14 ++-- llvm/utils/TableGen/Basic/CMakeLists.txt | 21 ++++++ .../{ => Basic}/CodeGenIntrinsics.cpp | 0 .../TableGen/{ => Basic}/CodeGenIntrinsics.h | 0 .../TableGen/{ => Basic}/SDNodeProperties.cpp | 0 .../TableGen/{ => Basic}/SDNodeProperties.h | 0 .../{ => Basic}/SequenceToOffsetTable.h | 0 llvm/utils/TableGen/CMakeLists.txt | 67 ++++++++----------- llvm/utils/TableGen/CallingConvEmitter.cpp | 2 +- llvm/utils/TableGen/CodeEmitterGen.cpp | 10 +-- llvm/utils/TableGen/CodeGenMapTable.cpp | 4 +- .../TableGen/{ => Common}/AsmWriterInst.cpp | 0 .../TableGen/{ => Common}/AsmWriterInst.h | 0 llvm/utils/TableGen/Common/CMakeLists.txt | 48 +++++++++++++ .../{ => Common}/CodeGenDAGPatterns.cpp | 0 .../{ => Common}/CodeGenDAGPatterns.h | 4 +- .../TableGen/{ => Common}/CodeGenHwModes.cpp | 0 .../TableGen/{ => Common}/CodeGenHwModes.h | 0 .../{ => Common}/CodeGenInstAlias.cpp | 0 .../TableGen/{ => Common}/CodeGenInstAlias.h | 0 .../{ => Common}/CodeGenInstruction.cpp | 0 .../{ => Common}/CodeGenInstruction.h | 0 .../{ => Common}/CodeGenRegisters.cpp | 0 .../TableGen/{ => Common}/CodeGenRegisters.h | 0 .../TableGen/{ => Common}/CodeGenSchedule.cpp | 0 .../TableGen/{ => Common}/CodeGenSchedule.h | 0 .../TableGen/{ => Common}/CodeGenTarget.cpp | 0 .../TableGen/{ => Common}/CodeGenTarget.h | 2 +- .../TableGen/{ => Common}/DAGISelMatcher.cpp | 0 .../TableGen/{ => Common}/DAGISelMatcher.h | 0 .../{ => Common}/GlobalISel/CXXPredicates.cpp | 0 .../{ => Common}/GlobalISel/CXXPredicates.h | 0 .../{ => Common}/GlobalISel/CodeExpander.cpp | 0 .../{ => Common}/GlobalISel/CodeExpander.h | 0 .../{ => Common}/GlobalISel/CodeExpansions.h | 0 .../{ => Common}/GlobalISel/CombinerUtils.h | 0 .../GlobalISel}/GlobalISelMatchTable.cpp | 4 +- .../GlobalISel}/GlobalISelMatchTable.h | 2 +- .../GlobalISelMatchTableExecutorEmitter.cpp | 0 .../GlobalISelMatchTableExecutorEmitter.h | 2 +- .../{ => Common}/GlobalISel/MatchDataInfo.cpp | 0 .../{ => Common}/GlobalISel/MatchDataInfo.h | 0 .../{ => Common}/GlobalISel/Patterns.cpp | 4 +- .../{ => Common}/GlobalISel/Patterns.h | 0 .../TableGen/{ => Common}/InfoByHwMode.cpp | 0 .../TableGen/{ => Common}/InfoByHwMode.h | 0 .../TableGen/{ => Common}/OptEmitter.cpp | 0 llvm/utils/TableGen/{ => Common}/OptEmitter.h | 0 .../{ => Common}/PredicateExpander.cpp | 0 .../TableGen/{ => Common}/PredicateExpander.h | 0 .../{ => Common}/SubtargetFeatureInfo.cpp | 0 .../{ => Common}/SubtargetFeatureInfo.h | 0 llvm/utils/TableGen/{ => Common}/Types.cpp | 0 llvm/utils/TableGen/{ => Common}/Types.h | 0 .../{ => Common}/VarLenCodeEmitterGen.cpp | 15 ++--- .../{ => Common}/VarLenCodeEmitterGen.h | 0 llvm/utils/TableGen/CompressInstEmitter.cpp | 6 +- llvm/utils/TableGen/DAGISelEmitter.cpp | 8 +-- llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/DAGISelMatcherGen.cpp | 14 ++-- llvm/utils/TableGen/DAGISelMatcherOpt.cpp | 6 +- llvm/utils/TableGen/DFAEmitter.cpp | 2 +- llvm/utils/TableGen/DFAPacketizerEmitter.cpp | 4 +- llvm/utils/TableGen/DXILEmitter.cpp | 4 +- llvm/utils/TableGen/DecoderEmitter.cpp | 10 +-- llvm/utils/TableGen/DisassemblerEmitter.cpp | 2 +- llvm/utils/TableGen/FastISelEmitter.cpp | 10 +-- llvm/utils/TableGen/GlobalISel/CMakeLists.txt | 20 ------ .../TableGen/GlobalISelCombinerEmitter.cpp | 24 +++---- llvm/utils/TableGen/GlobalISelEmitter.cpp | 18 ++--- llvm/utils/TableGen/InstrDocsEmitter.cpp | 6 +- llvm/utils/TableGen/InstrInfoEmitter.cpp | 16 ++--- llvm/utils/TableGen/IntrinsicEmitter.cpp | 4 +- .../TableGen/MacroFusionPredicatorEmitter.cpp | 4 +- llvm/utils/TableGen/OptParserEmitter.cpp | 2 +- llvm/utils/TableGen/OptRSTEmitter.cpp | 2 +- llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 4 +- llvm/utils/TableGen/RegisterBankEmitter.cpp | 6 +- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 12 ++-- .../utils/TableGen/SearchableTableEmitter.cpp | 4 +- llvm/utils/TableGen/SubtargetEmitter.cpp | 8 +-- .../WebAssemblyDisassemblerEmitter.cpp | 2 +- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86MnemonicTables.cpp | 4 +- llvm/utils/TableGen/X86RecognizableInstr.h | 2 +- 89 files changed, 238 insertions(+), 203 deletions(-) create mode 100644 llvm/utils/TableGen/Basic/CMakeLists.txt rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.h (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.h (100%) rename llvm/utils/TableGen/{ => Basic}/SequenceToOffsetTable.h (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.cpp (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.h (100%) create mode 100644 llvm/utils/TableGen/Common/CMakeLists.txt rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.h (99%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.h (99%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.cpp (100%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpansions.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CombinerUtils.h (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.cpp (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.h (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.h (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.cpp (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.h (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.cpp (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.h (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.h (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/Types.cpp (100%) rename llvm/utils/TableGen/{ => Common}/Types.h (100%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.cpp (97%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.h (100%) delete mode 100644 llvm/utils/TableGen/GlobalISel/CMakeLists.txt diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt index 7830e0218045..fae0eee06e4b 100644 --- a/llvm/unittests/TableGen/CMakeLists.txt +++ b/llvm/unittests/TableGen/CMakeLists.txt @@ -15,4 +15,4 @@ add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB ParserEntryPointTest.cpp ) -target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen) +target_link_libraries(TableGenTests PRIVATE LLVMTableGenCommon LLVMTableGen) diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp index 4a9a0e8c114b..1528884ffdf6 100644 --- a/llvm/unittests/TableGen/CodeExpanderTest.cpp +++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 0a2b1cf0f9a6..8b82ce899a48 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -95,12 +95,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SubtargetFeatureInfo.h" -#include "Types.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp index a27061ee585a..16661cd29edc 100644 --- a/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -11,13 +11,13 @@ // //===----------------------------------------------------------------------===// -#include "AsmWriterInst.h" -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/AsmWriterInst.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt new file mode 100644 index 000000000000..f2927d05c175 --- /dev/null +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -0,0 +1,21 @@ +# The basic TableGen library contains as little dependencies as possible. +# In particular, it does not depend on vt_gen -> it does not use ValueTypes. +# +# This library is the only thing included in `llvm-min-tablegen`. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL + CodeGenIntrinsics.cpp + SDNodeProperties.cpp +) +set_target_properties(LLVMTableGenBasic PROPERTIES FOLDER "Tablegenning") + +# Users may include its headers as "Basic/*.h" +target_include_directories(LLVMTableGenBasic + INTERFACE + $ + ) diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.cpp rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.h rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.h diff --git a/llvm/utils/TableGen/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.cpp rename to llvm/utils/TableGen/Basic/SDNodeProperties.cpp diff --git a/llvm/utils/TableGen/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.h rename to llvm/utils/TableGen/Basic/SDNodeProperties.h diff --git a/llvm/utils/TableGen/SequenceToOffsetTable.h b/llvm/utils/TableGen/Basic/SequenceToOffsetTable.h similarity index 100% rename from llvm/utils/TableGen/SequenceToOffsetTable.h rename to llvm/utils/TableGen/Basic/SequenceToOffsetTable.h diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 0100bf345ec2..14690329cabf 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -1,26 +1,25 @@ -add_subdirectory(GlobalISel) +# Basic utilities which is the strict minimum needed to build +# llvm-min-tblgen. +add_subdirectory(Basic) +# Common utilities are all of the reusable components and helper +# code needed by the backends. +add_subdirectory(Common) -add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL +set(LLVM_LINK_COMPONENTS Support) + +# llvm-min-tablegen only contains a subset of backends necessary to +# build llvm/include. It must not depend on TableGenCommon, as +# TableGenCommon depends on this already to generate things such as +# ValueType definitions. +add_tablegen(llvm-min-tblgen LLVM_HEADERS + TableGen.cpp Attributes.cpp - CodeGenIntrinsics.cpp DirectiveEmitter.cpp IntrinsicEmitter.cpp RISCVTargetDefEmitter.cpp - SDNodeProperties.cpp VTEmitter.cpp - PARTIAL_SOURCES_INTENDED - - LINK_COMPONENTS - Support - TableGen - ) -set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") + $ -set(LLVM_LINK_COMPONENTS Support) - -add_tablegen(llvm-min-tblgen LLVM_HEADERS - TableGen.cpp - $ PARTIAL_SOURCES_INTENDED ) set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning") @@ -35,63 +34,51 @@ add_tablegen(llvm-tblgen LLVM EXPORT LLVM AsmMatcherEmitter.cpp AsmWriterEmitter.cpp - AsmWriterInst.cpp - CTagsEmitter.cpp + Attributes.cpp CallingConvEmitter.cpp CodeEmitterGen.cpp - CodeGenDAGPatterns.cpp - CodeGenHwModes.cpp - CodeGenInstAlias.cpp - CodeGenInstruction.cpp CodeGenMapTable.cpp - CodeGenRegisters.cpp - CodeGenSchedule.cpp - CodeGenTarget.cpp + CompressInstEmitter.cpp + CTagsEmitter.cpp DAGISelEmitter.cpp DAGISelMatcherEmitter.cpp DAGISelMatcherGen.cpp DAGISelMatcherOpt.cpp - DAGISelMatcher.cpp DecoderEmitter.cpp DFAEmitter.cpp DFAPacketizerEmitter.cpp + DirectiveEmitter.cpp DisassemblerEmitter.cpp DXILEmitter.cpp ExegesisEmitter.cpp FastISelEmitter.cpp GlobalISelCombinerEmitter.cpp GlobalISelEmitter.cpp - GlobalISelMatchTable.cpp - GlobalISelMatchTableExecutorEmitter.cpp - InfoByHwMode.cpp - InstrInfoEmitter.cpp InstrDocsEmitter.cpp - OptEmitter.cpp + InstrInfoEmitter.cpp + IntrinsicEmitter.cpp + MacroFusionPredicatorEmitter.cpp OptParserEmitter.cpp OptRSTEmitter.cpp - PredicateExpander.cpp PseudoLoweringEmitter.cpp - CompressInstEmitter.cpp - MacroFusionPredicatorEmitter.cpp RegisterBankEmitter.cpp RegisterInfoEmitter.cpp + RISCVTargetDefEmitter.cpp SearchableTableEmitter.cpp SubtargetEmitter.cpp - SubtargetFeatureInfo.cpp TableGen.cpp - Types.cpp - VarLenCodeEmitterGen.cpp - X86DisassemblerTables.cpp + VTEmitter.cpp + WebAssemblyDisassemblerEmitter.cpp X86CompressEVEXTablesEmitter.cpp + X86DisassemblerTables.cpp X86FoldTablesEmitter.cpp X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp - WebAssemblyDisassemblerEmitter.cpp $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel) +target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp index 3c3a2874ce80..ec6ef56a66fa 100644 --- a/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp index 9194c13ccdcb..a57885f22d7e 100644 --- a/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -22,11 +22,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "VarLenCodeEmitterGen.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp index 03af0b49ba97..fbf1d47c0327 100644 --- a/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -75,8 +75,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" using namespace llvm; diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/Common/AsmWriterInst.cpp similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.cpp rename to llvm/utils/TableGen/Common/AsmWriterInst.cpp diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/Common/AsmWriterInst.h similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.h rename to llvm/utils/TableGen/Common/AsmWriterInst.h diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt new file mode 100644 index 000000000000..491d9bd2949d --- /dev/null +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -0,0 +1,48 @@ +# The common library is similar to the basic library except it can +# depend on vt_gen. +# +# This library contains the bulk of the supporting code for all +# TableGen backends. It's split off as a separate library to +# allow unit-testing those components. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL + GlobalISel/CodeExpander.cpp + GlobalISel/CXXPredicates.cpp + GlobalISel/GlobalISelMatchTable.cpp + GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp + GlobalISel/MatchDataInfo.cpp + GlobalISel/Patterns.cpp + + AsmWriterInst.cpp + CodeGenDAGPatterns.cpp + CodeGenHwModes.cpp + CodeGenInstAlias.cpp + CodeGenInstruction.cpp + CodeGenRegisters.cpp + CodeGenSchedule.cpp + CodeGenTarget.cpp + DAGISelMatcher.cpp + InfoByHwMode.cpp + OptEmitter.cpp + PredicateExpander.cpp + SubtargetFeatureInfo.cpp + Types.cpp + VarLenCodeEmitterGen.cpp + $ + + DEPENDS + vt_gen + ) +set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") +target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) + +# Users may include its headers as "Common/*.h" +target_include_directories(LLVMTableGenCommon + PUBLIC + $ + ) diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenDAGPatterns.cpp rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h similarity index 99% rename from llvm/utils/TableGen/CodeGenDAGPatterns.h rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.h index 823c40c922cb..7fcd39a9e940 100644 --- a/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h @@ -14,9 +14,9 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H -#include "CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SDNodeProperties.h" #include "CodeGenTarget.h" -#include "SDNodeProperties.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.cpp rename to llvm/utils/TableGen/Common/CodeGenHwModes.cpp diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/Common/CodeGenHwModes.h similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.h rename to llvm/utils/TableGen/Common/CodeGenHwModes.h diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.cpp rename to llvm/utils/TableGen/Common/CodeGenInstAlias.cpp diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/Common/CodeGenInstAlias.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.h rename to llvm/utils/TableGen/Common/CodeGenInstAlias.h diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.cpp rename to llvm/utils/TableGen/Common/CodeGenInstruction.cpp diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.h rename to llvm/utils/TableGen/Common/CodeGenInstruction.h diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.cpp rename to llvm/utils/TableGen/Common/CodeGenRegisters.cpp diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.h rename to llvm/utils/TableGen/Common/CodeGenRegisters.h diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.cpp rename to llvm/utils/TableGen/Common/CodeGenSchedule.cpp diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.h rename to llvm/utils/TableGen/Common/CodeGenSchedule.h diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenTarget.cpp rename to llvm/utils/TableGen/Common/CodeGenTarget.cpp diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h similarity index 99% rename from llvm/utils/TableGen/CodeGenTarget.h rename to llvm/utils/TableGen/Common/CodeGenTarget.h index e109c717dc01..df4c22ebb379 100644 --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/Common/CodeGenTarget.h @@ -16,10 +16,10 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#include "Basic/SDNodeProperties.h" #include "CodeGenHwModes.h" #include "CodeGenInstruction.h" #include "InfoByHwMode.h" -#include "SDNodeProperties.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.cpp rename to llvm/utils/TableGen/Common/DAGISelMatcher.cpp diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.h rename to llvm/utils/TableGen/Common/DAGISelMatcher.h diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.h rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpansions.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CombinerUtils.h rename to llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 45fb41b89f27..193f95443b16 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include "GlobalISelMatchTable.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" #include "llvm/Support/LEB128.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index b1ab7da8db44..aa86fad763d1 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H -#include "CodeGenDAGPatterns.h" +#include "Common/CodeGenDAGPatterns.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h index 7e952d6df309..d2b6a74c7577 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h @@ -15,7 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H -#include "SubtargetFeatureInfo.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.h rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISel/Patterns.cpp rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp index 758eac2dfebd..388bf7e9e833 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp @@ -7,11 +7,11 @@ //===----------------------------------------------------------------------===// #include "Patterns.h" -#include "../CodeGenInstruction.h" -#include "../CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" #include "CXXPredicates.h" #include "CodeExpander.h" #include "CodeExpansions.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/Patterns.h rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.h diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.cpp rename to llvm/utils/TableGen/Common/InfoByHwMode.cpp diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.h rename to llvm/utils/TableGen/Common/InfoByHwMode.h diff --git a/llvm/utils/TableGen/OptEmitter.cpp b/llvm/utils/TableGen/Common/OptEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/OptEmitter.cpp rename to llvm/utils/TableGen/Common/OptEmitter.cpp diff --git a/llvm/utils/TableGen/OptEmitter.h b/llvm/utils/TableGen/Common/OptEmitter.h similarity index 100% rename from llvm/utils/TableGen/OptEmitter.h rename to llvm/utils/TableGen/Common/OptEmitter.h diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.cpp rename to llvm/utils/TableGen/Common/PredicateExpander.cpp diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/Common/PredicateExpander.h similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.h rename to llvm/utils/TableGen/Common/PredicateExpander.h diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.cpp rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.h similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.h rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.h diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Common/Types.cpp similarity index 100% rename from llvm/utils/TableGen/Types.cpp rename to llvm/utils/TableGen/Common/Types.cpp diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Common/Types.h similarity index 100% rename from llvm/utils/TableGen/Types.h rename to llvm/utils/TableGen/Common/Types.h diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp similarity index 97% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.cpp rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp index bfb7e5c33317..4263d8f41715 100644 --- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp +++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp @@ -337,8 +337,8 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits, return; } - IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " - << "/*Index*/" << Index << "},"; + IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " << "/*Index*/" + << Index << "},"; SS.indent(4); for (unsigned I = 0; I < Bits.getNumWords(); ++I, ++Index) @@ -371,8 +371,8 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( if (ModeIt == InstIt->second.end()) ModeIt = InstIt->second.find(Universal); if (ModeIt == InstIt->second.end()) { - IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" - << "// " << R->getName() << " no encoding\n"; + IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" << "// " << R->getName() + << " no encoding\n"; continue; } const VarLenInst &VLI = ModeIt->second; @@ -492,10 +492,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n"; - SS.indent(I) << "Inst.insertBits(" - << "Scratch.extractBits(" << utostr(NumBits) << ", " - << utostr(LoBit) << ")" - << ", " << Offset << ");\n"; + SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits(" + << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", " + << Offset << ");\n"; HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit); } diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h similarity index 100% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.h rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp index f703fff0ef3e..fcf77934faac 100644 --- a/llvm/utils/TableGen/CompressInstEmitter.cpp +++ b/llvm/utils/TableGen/CompressInstEmitter.cpp @@ -64,9 +64,9 @@ //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp index 336cee09b90c..b43a8e659dd9 100644 --- a/llvm/utils/TableGen/DAGISelEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 533b8c423690..dcecac4380ce 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -10,12 +10,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index e8bdabaa0c7e..99babdf07316 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "InfoByHwMode.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 047d285f9914..224102e49d98 100644 --- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp index ce8cc2a078d7..567184d3d5ee 100644 --- a/llvm/utils/TableGen/DFAEmitter.cpp +++ b/llvm/utils/TableGen/DFAEmitter.cpp @@ -21,7 +21,7 @@ //===----------------------------------------------------------------------===// #include "DFAEmitter.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/UniqueVector.h" diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp index 26ea1846ffae..3c74df048660 100644 --- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp +++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -14,8 +14,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" #include "DFAEmitter.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index af1efb8aa99f..47c89df35e19 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 732f34ed04c5..494dc93faace 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "TableGenBackends.h" -#include "VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/CachedHashString.h" diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp index 2d653af4d302..d41750075b41 100644 --- a/llvm/utils/TableGen/DisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "TableGenBackends.h" #include "WebAssemblyDisassemblerEmitter.h" #include "X86DisassemblerTables.h" diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp index f04c6e3b3bf0..acfdc20316b7 100644 --- a/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/llvm/utils/TableGen/FastISelEmitter.cpp @@ -16,11 +16,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt deleted file mode 100644 index 7262c4058399..000000000000 --- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(LLVM_LINK_COMPONENTS - Support - TableGen - ) - -add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB - CodeExpander.cpp - CXXPredicates.cpp - MatchDataInfo.cpp - Patterns.cpp - - DEPENDS - vt_gen - ) - -# Users may include its headers as "GlobalISel/*.h" -target_include_directories(LLVMTableGenGlobalISel - INTERFACE - $ - ) diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index dee3cb4d71a4..39b9f8a2ae17 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -26,18 +26,18 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" -#include "GlobalISel/CXXPredicates.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISel/MatchDataInfo.h" -#include "GlobalISel/Patterns.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/CXXPredicates.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CombinerUtils.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/GlobalISel/MatchDataInfo.h" +#include "Common/GlobalISel/Patterns.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/Hashing.h" diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index e86057422cd7..25e302ce1ca4 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -30,15 +30,15 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "InfoByHwMode.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/InfoByHwMode.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGenTypes/LowLevelType.h" #include "llvm/CodeGenTypes/MachineValueType.h" diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp index efabf6bb7ba6..f948540e18db 100644 --- a/llvm/utils/TableGen/InstrDocsEmitter.cpp +++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -18,9 +18,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp index 2d08447429d9..36f8fa146539 100644 --- a/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -11,15 +11,15 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" -#include "SequenceToOffsetTable.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "TableGenBackends.h" -#include "Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 50a34eac7ca3..a7e99fa4c050 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "SequenceToOffsetTable.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index 91c3b0b4359c..e9e63fa8d0de 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -38,8 +38,8 @@ // //===---------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp index c25f6c59cab3..6334af53f88f 100644 --- a/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/llvm/utils/TableGen/OptParserEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp index 5a7f079dc168..75b7cbdf2988 100644 --- a/llvm/utils/TableGen/OptRSTEmitter.cpp +++ b/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp index 7f692f29192d..01cfd4a1d982 100644 --- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp +++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp index 8b59411c5bc3..5546e727af38 100644 --- a/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/BitVector.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index c4fc1930488c..a1259bff6ba8 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp index 51f18f360ed3..48ee23db957d 100644 --- a/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp index d350d7de139f..2e2c57b802ee 100644 --- a/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp index 928129f24fcb..e9436ab16e44 100644 --- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -14,7 +14,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyDisassemblerEmitter.h" -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 0a9abbfe186e..c721502a395f 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -11,8 +11,8 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 1319042e48d0..5871e678b16e 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FormattedStream.h" diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp index aeafee157462..d9ceed40f7c7 100644 --- a/llvm/utils/TableGen/X86MnemonicTables.cpp +++ b/llvm/utils/TableGen/X86MnemonicTables.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 68af68fb5aa0..12fb41750cb3 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/Support/X86DisassemblerDecoderCommon.h" #include #include -- GitLab From aa962d67ee896f416e285a9298e45fc08ff95eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:42:41 +0100 Subject: [PATCH 022/343] [clang-repl] Fix Value for platforms where unqualified char is unsigned (#86118) Signedness of unqualified `char` is unspecified and varies between platforms. This patch adds `Char_U` in `REPL_BUILTIN_TYPES` to account for platforms that default to `unsigned char`. --- clang/include/clang/Interpreter/Value.h | 1 + clang/unittests/Interpreter/InterpreterTest.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h index c380cd91550d..d70e8f871902 100644 --- a/clang/include/clang/Interpreter/Value.h +++ b/clang/include/clang/Interpreter/Value.h @@ -76,6 +76,7 @@ class QualType; X(bool, Bool) \ X(char, Char_S) \ X(signed char, SChar) \ + X(unsigned char, Char_U) \ X(unsigned char, UChar) \ X(short, Short) \ X(unsigned short, UShort) \ diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index e76c0677db5e..69bc2da24288 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -340,6 +340,12 @@ TEST(InterpreterTest, Value) { EXPECT_EQ(V1.getKind(), Value::K_Int); EXPECT_FALSE(V1.isManuallyAlloc()); + Value V1b; + llvm::cantFail(Interp->ParseAndExecute("char c = 42;")); + llvm::cantFail(Interp->ParseAndExecute("c", &V1b)); + EXPECT_TRUE(V1b.getKind() == Value::K_Char_S || + V1b.getKind() == Value::K_Char_U); + Value V2; llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;")); llvm::cantFail(Interp->ParseAndExecute("y", &V2)); -- GitLab From 0cf4788d9d0df60980cb48d28aafe7a86aa15a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:44:25 +0100 Subject: [PATCH 023/343] [clang-repl] Factor out CreateJITBuilder() and allow specialization in derived classes (#84461) The LLJITBuilder interface provides a very convenient way to configure the ORCv2 JIT engine. IncrementalExecutor already used it internally to construct the JIT, but didn't provide external access. This patch lifts control of the creation process to the Interpreter and allows injection of a custom instance through the extended interface. The Interpreter's default behavior remains unchanged and the IncrementalExecutor remains an implementation detail. --- clang/include/clang/Interpreter/Interpreter.h | 8 ++ clang/lib/Interpreter/IncrementalExecutor.cpp | 33 ++--- clang/lib/Interpreter/IncrementalExecutor.h | 9 +- clang/lib/Interpreter/Interpreter.cpp | 26 +++- .../Interpreter/InterpreterExtensionsTest.cpp | 121 +++++++++++++++++- 5 files changed, 175 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 1dcba1ef9679..970e0245417b 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -30,6 +30,7 @@ namespace llvm { namespace orc { class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -127,6 +128,13 @@ protected: // custom runtime. virtual std::unique_ptr FindRuntimeInterface(); + // Lazily construct thev ORCv2 JITBuilder. This called when the internal + // IncrementalExecutor is created. The default implementation populates an + // in-process JIT with debugging support. Override this to configure the JIT + // engine used for execution. + virtual llvm::Expected> + CreateJITBuilder(CompilerInstance &CI); + public: virtual ~Interpreter(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 40bcef94797d..6f036107c14a 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" @@ -36,26 +37,28 @@ LLVM_ATTRIBUTE_USED void linkComponents() { namespace clang { +llvm::Expected> +IncrementalExecutor::createDefaultJITBuilder( + llvm::orc::JITTargetMachineBuilder JTMB) { + auto JITBuilder = std::make_unique(); + JITBuilder->setJITTargetMachineBuilder(std::move(JTMB)); + JITBuilder->setPrePlatformSetup([](llvm::orc::LLJIT &J) { + // Try to enable debugging of JIT'd code (only works with JITLink for + // ELF and MachO). + consumeError(llvm::orc::enableDebuggerSupport(J)); + return llvm::Error::success(); + }); + return std::move(JITBuilder); +} + IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, - llvm::Error &Err, - const clang::TargetInfo &TI) + llvm::orc::LLJITBuilder &JITBuilder, + llvm::Error &Err) : TSCtx(TSC) { using namespace llvm::orc; llvm::ErrorAsOutParameter EAO(&Err); - auto JTMB = JITTargetMachineBuilder(TI.getTriple()); - JTMB.addFeatures(TI.getTargetOpts().Features); - LLJITBuilder Builder; - Builder.setJITTargetMachineBuilder(JTMB); - Builder.setPrePlatformSetup( - [](LLJIT &J) { - // Try to enable debugging of JIT'd code (only works with JITLink for - // ELF and MachO). - consumeError(enableDebuggerSupport(J)); - return llvm::Error::success(); - }); - - if (auto JitOrErr = Builder.create()) + if (auto JitOrErr = JITBuilder.create()) Jit = std::move(*JitOrErr); else { Err = JitOrErr.takeError(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index dd0a210a0614..b4347209e14f 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -23,7 +23,9 @@ namespace llvm { class Error; namespace orc { +class JITTargetMachineBuilder; class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -44,8 +46,8 @@ class IncrementalExecutor { public: enum SymbolNameKind { IRName, LinkerName }; - IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err, - const clang::TargetInfo &TI); + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); ~IncrementalExecutor(); llvm::Error addModule(PartialTranslationUnit &PTU); @@ -56,6 +58,9 @@ public: getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const; llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + + static llvm::Expected> + createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); }; } // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 7fa52f2f15fc..cf31456b6950 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -372,15 +372,35 @@ Interpreter::Parse(llvm::StringRef Code) { return IncrParser->Parse(Code); } +static llvm::Expected +createJITTargetMachineBuilder(const std::string &TT) { + if (TT == llvm::sys::getProcessTriple()) + // This fails immediately if the target backend is not registered + return llvm::orc::JITTargetMachineBuilder::detectHost(); + + // If the target backend is not registered, LLJITBuilder::create() will fail + return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); +} + +llvm::Expected> +Interpreter::CreateJITBuilder(CompilerInstance &CI) { + auto JTMB = createJITTargetMachineBuilder(CI.getTargetOpts().Triple); + if (!JTMB) + return JTMB.takeError(); + return IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); +} + llvm::Error Interpreter::CreateExecutor() { - const clang::TargetInfo &TI = - getCompilerInstance()->getASTContext().getTargetInfo(); if (IncrExecutor) return llvm::make_error("Operation failed. " "Execution engine exists", std::error_code()); + llvm::Expected> JB = + CreateJITBuilder(*getCompilerInstance()); + if (!JB) + return JB.takeError(); llvm::Error Err = llvm::Error::success(); - auto Executor = std::make_unique(*TSCtx, Err, TI); + auto Executor = std::make_unique(*TSCtx, **JB, Err); if (!Err) IncrExecutor = std::move(Executor); diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index b7708616fd24..8bc429d9ec2d 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -18,14 +18,21 @@ #include "clang/Sema/Sema.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/Support/Error.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Threading.h" #include "llvm/Testing/Support/Error.h" #include "gmock/gmock.h" #include "gtest/gtest.h" + #include +#if defined(_AIX) +#define CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +#endif + using namespace clang; namespace { @@ -41,6 +48,10 @@ struct LLVMInitRAII { LLVMInitRAII() { llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmPrinter(); + LLVMInitializeARMTarget(); + LLVMInitializeARMTargetInfo(); + LLVMInitializeARMTargetMC(); + LLVMInitializeARMAsmPrinter(); } ~LLVMInitRAII() { llvm::llvm_shutdown(); } } LLVMInit; @@ -51,12 +62,30 @@ public: llvm::Error &Err) : Interpreter(std::move(CI), Err) {} - llvm::Error testCreateExecutor() { return Interpreter::CreateExecutor(); } + llvm::Error testCreateJITBuilderError() { + JB = nullptr; + return Interpreter::CreateExecutor(); + } + + llvm::Error testCreateExecutor() { + JB = std::make_unique(); + return Interpreter::CreateExecutor(); + } void resetExecutor() { Interpreter::ResetExecutor(); } + +private: + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JB) + return std::move(JB); + return llvm::make_error("TestError", std::error_code()); + } + + std::unique_ptr JB; }; -#ifdef _AIX +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { #else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { @@ -69,6 +98,8 @@ TEST(InterpreterExtensionsTest, ExecutorCreateReset) { llvm::Error ErrOut = llvm::Error::success(); TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut); cantFail(std::move(ErrOut)); + EXPECT_THAT_ERROR(Interp.testCreateJITBuilderError(), + llvm::FailedWithMessage("TestError")); cantFail(Interp.testCreateExecutor()); Interp.resetExecutor(); cantFail(Interp.testCreateExecutor()); @@ -126,4 +157,90 @@ TEST(InterpreterExtensionsTest, FindRuntimeInterface) { EXPECT_EQ(1U, Interp.RuntimeIBPtr->TransformerQueries); } +class CustomJBInterpreter : public Interpreter { + using CustomJITBuilderCreatorFunction = + std::function>()>; + CustomJITBuilderCreatorFunction JBCreator = nullptr; + +public: + CustomJBInterpreter(std::unique_ptr CI, llvm::Error &ErrOut) + : Interpreter(std::move(CI), ErrOut) {} + + ~CustomJBInterpreter() override { + // Skip cleanUp() because it would trigger LLJIT default dtors + Interpreter::ResetExecutor(); + } + + void setCustomJITBuilderCreator(CustomJITBuilderCreatorFunction Fn) { + JBCreator = std::move(Fn); + } + + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JBCreator) + return JBCreator(); + return Interpreter::CreateJITBuilder(CI); + } + + llvm::Error CreateExecutor() { return Interpreter::CreateExecutor(); } +}; + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) { +#else +TEST(InterpreterExtensionsTest, DefaultCrossJIT) { +#endif + IncrementalCompilerBuilder CB; + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + cantFail(Interp.CreateExecutor()); +} + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) { +#else +TEST(InterpreterExtensionsTest, CustomCrossJIT) { +#endif + std::string TargetTriple = "armv6-none-eabi"; + + IncrementalCompilerBuilder CB; + CB.SetTargetTriple(TargetTriple); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + + using namespace llvm::orc; + LLJIT *JIT = nullptr; + std::vector> Objs; + Interp.setCustomJITBuilderCreator([&]() { + auto JTMB = JITTargetMachineBuilder(llvm::Triple(TargetTriple)); + JTMB.setCPU("cortex-m0plus"); + auto JB = std::make_unique(); + JB->setJITTargetMachineBuilder(JTMB); + JB->setPlatformSetUp(setUpInactivePlatform); + JB->setNotifyCreatedCallback([&](LLJIT &J) { + ObjectLayer &ObjLayer = J.getObjLinkingLayer(); + auto *JITLinkObjLayer = llvm::dyn_cast(&ObjLayer); + JITLinkObjLayer->setReturnObjectBuffer( + [&Objs](std::unique_ptr MB) { + Objs.push_back(std::move(MB)); + }); + JIT = &J; + return llvm::Error::success(); + }); + return JB; + }); + + EXPECT_EQ(0U, Objs.size()); + cantFail(Interp.CreateExecutor()); + cantFail(Interp.ParseAndExecute("int a = 1;")); + ExecutorAddr Addr = cantFail(JIT->lookup("a")); + EXPECT_NE(0U, Addr.getValue()); + EXPECT_EQ(1U, Objs.size()); +} + } // end anonymous namespace -- GitLab From 75e528fdd9594ecb6fdb5d9e7bee1506f7e43be0 Mon Sep 17 00:00:00 2001 From: David Stuttard Date: Mon, 25 Mar 2024 09:01:46 +0000 Subject: [PATCH 024/343] [AMDGPU] Extend zero initialization of return values for TFE (#85759) buffer_load instructions that use TFE also need to zero initialize return values similar to how the image instructions currently work. Add support for this with standard zero init of all results + zero init of just TFE flag when enable-prt-strict-null subtarget feature is disabled. --- .../AMDGPU/AMDGPUInstructionSelector.cpp | 32 -- llvm/lib/Target/AMDGPU/BUFInstructions.td | 5 +- llvm/lib/Target/AMDGPU/MIMGInstructions.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 87 ++--- llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 6 + llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 + .../llvm.amdgcn.struct.buffer.load.format.ll | 1 + ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 1 + .../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 111 ++++-- .../llvm.amdgcn.struct.buffer.load.format.ll | 317 +++++++++++++++++- ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 280 ++++++++++++++++ 12 files changed, 741 insertions(+), 105 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 94cc1d90e0ca..e13c13913d4e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -2045,38 +2045,6 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic( if (BaseOpcode->HasD16) MIB.addImm(IsD16 ? -1 : 0); - if (IsTexFail) { - // An image load instruction with TFE/LWE only conditionally writes to its - // result registers. Initialize them to zero so that we always get well - // defined result values. - assert(VDataOut && !VDataIn); - Register Tied = MRI->cloneVirtualRegister(VDataOut); - Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) - .addImm(0); - auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); - if (STI.usePRTStrictNull()) { - // With enable-prt-strict-null enabled, initialize all result registers to - // zero. - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts) - RegSeq.addReg(Zero).addImm(Sub); - } else { - // With enable-prt-strict-null disabled, only initialize the extra TFE/LWE - // result register. - Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts.drop_back(1)) - RegSeq.addReg(Undef).addImm(Sub); - RegSeq.addReg(Zero).addImm(Parts.back()); - } - MIB.addReg(Tied, RegState::Implicit); - MIB->tieOperands(0, MIB->getNumOperands() - 1); - } - MI.eraseFromParent(); constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 4ae514ffcf78..273f92abf354 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -86,7 +86,7 @@ class BUF_Pseudo has_soffset = 1; bits<1> has_offset = 1; bits<1> has_slc = 1; - bits<1> tfe = ?; + bits<1> tfe = 0; bits<4> elements = 0; bits<1> has_sccb = 1; bits<1> sccb_value = 0; @@ -323,6 +323,7 @@ class MUBUF_Pseudo (MUBUFGetBaseOpcode.ret); let MUBUF = 1; let AsmMatchConverter = "cvtMubuf"; + let usesCustomInserter = 1; } class MUBUF_Real : @@ -3369,7 +3370,7 @@ def MUBUFInfoTable : GenericTable { let CppTypeName = "MUBUFInfo"; let Fields = [ "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", - "IsBufferInv" + "IsBufferInv", "tfe" ]; let PrimaryKey = ["Opcode"]; diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index 595ef39ce03e..23e8be0d5e45 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -210,6 +210,7 @@ class MIMG : MIMG_Base { let hasPostISelHook = 1; + let usesCustomInserter = 1; Instruction Opcode = !cast(NAME); MIMGBaseOpcode BaseOpcode; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d437f339a687..81a231f0cade 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -5410,6 +5410,11 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( return SplitBB; } default: + if (TII->isImage(MI) || TII->isMUBUF(MI)) { + if (!MI.mayStore()) + AddMemOpInit(MI); + return BB; + } return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); } } @@ -15034,60 +15039,67 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node, // result register that will be written in the case of a memory access failure. // The required code is also added to tie this init code to the result of the // img instruction. -void SITargetLowering::AddIMGInit(MachineInstr &MI) const { +void SITargetLowering::AddMemOpInit(MachineInstr &MI) const { const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo(); MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); MachineBasicBlock &MBB = *MI.getParent(); - MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); - MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); - MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - - if (!TFE && !LWE) // intersect_ray - return; + int DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + unsigned InitIdx = 0; - unsigned TFEVal = TFE ? TFE->getImm() : 0; - unsigned LWEVal = LWE ? LWE->getImm() : 0; - unsigned D16Val = D16 ? D16->getImm() : 0; + if (TII->isImage(MI)) { + MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); + MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); + MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - if (!TFEVal && !LWEVal) - return; + if (!TFE && !LWE) // intersect_ray + return; - // At least one of TFE or LWE are non-zero - // We have to insert a suitable initialization of the result value and - // tie this to the dest of the image instruction. + unsigned TFEVal = TFE ? TFE->getImm() : 0; + unsigned LWEVal = LWE ? LWE->getImm() : 0; + unsigned D16Val = D16 ? D16->getImm() : 0; - const DebugLoc &DL = MI.getDebugLoc(); + if (!TFEVal && !LWEVal) + return; - int DstIdx = - AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + // At least one of TFE or LWE are non-zero + // We have to insert a suitable initialization of the result value and + // tie this to the dest of the image instruction. - // Calculate which dword we have to initialize to 0. - MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); + // Calculate which dword we have to initialize to 0. + MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); - // check that dmask operand is found. - assert(MO_Dmask && "Expected dmask operand in instruction"); + // check that dmask operand is found. + assert(MO_Dmask && "Expected dmask operand in instruction"); - unsigned dmask = MO_Dmask->getImm(); - // Determine the number of active lanes taking into account the - // Gather4 special case - unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); + unsigned dmask = MO_Dmask->getImm(); + // Determine the number of active lanes taking into account the + // Gather4 special case + unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); - bool Packed = !Subtarget->hasUnpackedD16VMem(); + bool Packed = !Subtarget->hasUnpackedD16VMem(); - unsigned InitIdx = - D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; + InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; - // Abandon attempt if the dst size isn't large enough - // - this is in fact an error but this is picked up elsewhere and - // reported correctly. - uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; - if (DstSize < InitIdx) + // Abandon attempt if the dst size isn't large enough + // - this is in fact an error but this is picked up elsewhere and + // reported correctly. + uint32_t DstSize = + TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + if (DstSize < InitIdx) + return; + } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) { + InitIdx = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + } else { return; + } + + const DebugLoc &DL = MI.getDebugLoc(); // Create a register for the initialization value. - Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg()); unsigned NewDst = 0; // Final initialized value will be in here // If PRTStrictNull feature is enabled (the default) then initialize @@ -15185,11 +15197,8 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, return; } - if (TII->isImage(MI)) { - if (!MI.mayStore()) - AddIMGInit(MI); + if (TII->isImage(MI)) TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr); - } } static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 89da4428e3ab..9856a2923d38 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -466,7 +466,7 @@ public: SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override; - void AddIMGInit(MachineInstr &MI) const; + void AddMemOpInit(MachineInstr &MI) const; void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..a90dc32d396f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -318,6 +318,7 @@ struct MUBUFInfo { bool has_srsrc; bool has_soffset; bool IsBufferInv; + bool tfe; }; struct MTBUFInfo { @@ -466,6 +467,11 @@ bool getMUBUFIsBufferInv(unsigned Opc) { return Info ? Info->IsBufferInv : false; } +bool getMUBUFTfe(unsigned Opc) { + const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); + return Info ? Info->tfe : false; +} + bool getSMEMIsBuffer(unsigned Opc) { const SMInfo *Info = getSMEMOpcodeHelper(Opc); return Info ? Info->IsBuffer : false; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..f4f9a787100b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -525,6 +525,9 @@ bool getMUBUFHasSoffset(unsigned Opc); LLVM_READONLY bool getMUBUFIsBufferInv(unsigned Opc); +LLVM_READONLY +bool getMUBUFTfe(unsigned Opc); + LLVM_READONLY bool getSMEMIsBuffer(unsigned Opc); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll index 686b849ff58f..06bd45a45cce 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX12 %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; GFX8-LABEL: name: struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll index 9edc24554911..1e3f94a5e39c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; CHECK-LABEL: name: struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll index 1348315e72e7..7b1f55e7eeba 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll @@ -22,18 +22,36 @@ main_body: define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_both: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x00,0x00,0x60,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x05] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v9, v8 ; encoding: [0x08,0x03,0x12,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v10, v8 ; encoding: [0x08,0x03,0x14,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v11, v8 ; encoding: [0x08,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v8 ; encoding: [0x08,0x03,0x18,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:7], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x05,0x00,0x60,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x08,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_both: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x05] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0a,0x09] +; GFX12-NEXT: v_dual_mov_b32 v11, v8 :: v_dual_mov_b32 v12, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0c,0x0b] +; GFX12-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v7, v6, v5], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x07,0x06,0x05,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x08,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32i32.i32(i32 2, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0) @@ -63,18 +81,37 @@ main_body: define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, v3 ; encoding: [0x80,0x00,0x10,0xca,0x03,0x01,0x08,0x09] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v10, v9 ; encoding: [0x00,0x01,0x10,0xca,0x09,0x01,0x0a,0x05] +; GFX11-NEXT: v_mov_b32_e32 v11, v9 ; encoding: [0x09,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v9 ; encoding: [0x09,0x03,0x18,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v13, v9 ; encoding: [0x09,0x03,0x1a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:8], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x05,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x09,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2, v3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_mov_b32_e32 v9, 0 ; encoding: [0x80,0x02,0x12,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v6, v2 ; encoding: [0x03,0x01,0x10,0xca,0x02,0x01,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v0 ; encoding: [0x01,0x01,0x10,0xca,0x00,0x01,0x08,0x07] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x23,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v10, v9 :: v_dual_mov_b32 v11, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0a,0x0a] +; GFX12-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0c,0x0c] +; GFX12-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v8, v7, v6, v5], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x08,0x07,0x06,0x05] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x09,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32i32.i32(i32 8, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -155,18 +192,31 @@ main_body: define amdgpu_ps <4 x half> @load_2dmsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x03] +; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x05] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], v[3:5], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x03,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x06,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x03] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v6 ; encoding: [0x06,0x01,0x10,0xca,0x06,0x01,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x05,0x04,0x03,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x06,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -196,18 +246,31 @@ main_body: define amdgpu_ps <4 x half> @load_2darraymsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9c,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX11-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9d,0x01,0x62,0xf0,0x06,0x00,0x20,0x00,0x05,0x04,0x03,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x07,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX12-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7 ; encoding: [0x07,0x01,0x10,0xca,0x07,0x01,0x08,0x08] +; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x06,0x05,0x04,0x03] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x07,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll index 00be32b06de0..ba3d306cc0cf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-enable-prt-strict-null -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s ;RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s @@ -34,6 +35,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -75,6 +86,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -146,6 +164,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) { ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs_large: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -196,6 +233,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_12bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -235,6 +279,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_13bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -274,6 +327,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_16bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -313,6 +375,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_23bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -352,6 +423,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-SDAG-LABEL: buffer_load_voffset_large_24bit: ; GFX12-SDAG: ; %bb.0: ; %main_body ; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0x800000 :: v_dual_mov_b32 v0, 0 @@ -389,6 +469,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_idx: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], null idxen @@ -427,6 +513,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -466,6 +561,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs_imm: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -497,6 +601,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], null idxen offen @@ -529,6 +639,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i3 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both_reversed: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v2, v0 @@ -562,6 +679,13 @@ define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -595,6 +719,13 @@ define amdgpu_ps float @buffer_load_x_i32(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x_i32: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -629,6 +760,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_xy: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -644,7 +782,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 -; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX6-NEXT: v_mov_b32_e32 v7, 2 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 +; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s0, s2 @@ -658,7 +801,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 -; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX8PLUS-NEXT: v_mov_b32_e32 v7, 2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 +; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX8PLUS-NEXT: v_mov_b32_e32 v0, v6 @@ -667,22 +815,40 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 +; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v2, 2 +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4i32_tfe: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v2, 0 -; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe +; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 +; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX12-NEXT: v_mov_b32_e32 v0, v6 ; GFX12-NEXT: ; return to shader part epilog - %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0) + %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out %status = extractvalue { <4 x i32>, i32 } %load, 1 @@ -694,6 +860,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -708,6 +878,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -718,15 +892,32 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 ; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off @@ -744,6 +935,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -759,6 +953,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -769,15 +966,31 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -795,6 +1008,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -810,6 +1026,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -820,15 +1039,31 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -846,6 +1081,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -860,6 +1098,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -870,15 +1110,29 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -896,6 +1150,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -910,6 +1167,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -920,15 +1179,29 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -946,6 +1219,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -960,6 +1234,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -970,15 +1245,28 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off @@ -996,6 +1284,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -1010,6 +1299,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -1020,15 +1310,28 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll index b0bd4e428ef2..c5202b84fa1e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mattr=-enable-prt-strict-null -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) { ; GFX6-LABEL: buffer_load: @@ -31,6 +32,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp ; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0) %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1) @@ -62,6 +73,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 42, i32 0, i32 0) ret <4 x float> %data @@ -126,6 +144,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg) ; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog main_body: %d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0) %d.1 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 32764, i32 0) @@ -156,6 +193,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 0, i32 0) ret <4 x float> %data @@ -188,6 +232,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8188, i32 0, i32 0) ret <4 x float> %data @@ -220,6 +273,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 65532, i32 0, i32 0) ret <4 x float> %data @@ -252,6 +314,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8388604, i32 0, i32 0) ret <4 x float> %data @@ -284,6 +355,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 16777212, i32 0, i32 0) ret <4 x float> %data @@ -307,6 +387,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0, i32 0) ret <4 x float> %data @@ -339,6 +425,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -371,6 +466,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %ofs = add i32 %1, 60 %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %ofs, i32 0, i32 0) @@ -395,6 +499,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(ptr addrspace(8) inreg, i32, i32) ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 %2, i32 0, i32 0) ret <4 x float> %data @@ -421,6 +531,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(ptr addrspace(8) inreg, ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %2, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -447,6 +564,13 @@ define amdgpu_ps float @buffer_load_x(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret float %data @@ -473,6 +597,13 @@ define amdgpu_ps float @buffer_load_x_i32(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call i32 @llvm.amdgcn.struct.ptr.buffer.load.format.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %fdata = bitcast i32 %data to float @@ -500,6 +631,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret <2 x float> %data @@ -509,6 +647,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -523,6 +665,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -533,11 +679,25 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out @@ -550,6 +710,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -564,6 +728,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -574,11 +742,25 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x float>, i32 } %load, 0 store <4 x float> %data, ptr addrspace(1) %out @@ -591,6 +773,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -606,6 +791,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -616,11 +804,24 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x i32>, i32 } %load, 0 store <3 x i32> %data, ptr addrspace(1) %out @@ -633,6 +834,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -648,6 +852,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -658,11 +865,24 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x float>, i32 } %load, 0 store <3 x float> %data, ptr addrspace(1) %out @@ -675,6 +895,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -689,6 +912,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -699,11 +924,23 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x i32>, i32 } %load, 0 store <2 x i32> %data, ptr addrspace(1) %out @@ -716,6 +953,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -730,6 +970,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -740,11 +982,23 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x float>, i32 } %load, 0 store <2 x float> %data, ptr addrspace(1) %out @@ -757,6 +1011,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -771,6 +1026,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -781,11 +1037,22 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { i32, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { i32, i32 } %load, 0 store i32 %data, ptr addrspace(1) %out @@ -798,6 +1065,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -812,6 +1080,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -822,11 +1091,22 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { float, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { float, i32 } %load, 0 store float %data, ptr addrspace(1) %out -- GitLab From 99c40f6ba60ea6e5dbd3f015956c7d2f6e25e54c Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:13:42 +0100 Subject: [PATCH 025/343] [SPIR-V] Introduce a command line option to support compatibility with Khronos SPIRV Translator (#86101) SPIRV-LLVM-Translator project (https://github.com/KhronosGroup/SPIRV-LLVM-Translator) from Khronos Group is a tool and a library for bi-directional translation between SPIR-V and LLVM IR. In its backward translation from SPIR-V to LLVM IR SPIRV-LLVM-Translator isn't necessarily able to cover the same SPIR-V patterns/instructions set that SPIRV Backend produces, even if we target the same SPIR-V version in both SPIRV-LLVM-Translator and SPIRV Backend projects. To improve interoperability and ability to apply SPIRV Backend output in different products this PR introduces a notion of a mode of SPIR-V output that is compatible with a subset of SPIR-V supported by SPIRV-LLVM-Translator. This includes a new command line option that doesn't influence default behavior of SPIRV Backend and one test case that demonstrates how this command line option may be used to get a practical benefit of producing that one of two possible and similar output options that can be understood by SPIRV-LLVM-Translator. --- llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp | 8 +++++++- llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp index 38caa7c8ea0a..09a029a35a74 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -27,6 +27,11 @@ using namespace llvm; #define GET_SUBTARGETINFO_CTOR #include "SPIRVGenSubtargetInfo.inc" +static cl::opt + SPVTranslatorCompat("translator-compatibility-mode", + cl::desc("SPIR-V Translator compatibility mode"), + cl::Optional, cl::init(false)); + cl::list Extensions( "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore, cl::Hidden, @@ -157,8 +162,9 @@ bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const { } // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual. +// In SPIR-V Translator compatibility mode this feature is not available. bool SPIRVSubtarget::canDirectlyComparePointers() const { - return isAtLeastVer(SPIRVVersion, 14); + return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, 14); } void SPIRVSubtarget::initAvailableExtensions() { diff --git a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll index 641e2bf0649c..31cd8bd45929 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll @@ -1,7 +1,13 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - | FileCheck %s --check-prefix=CHECK-COMPAT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[EQ:%.*]] "test_eq" ; CHECK-DAG: OpName [[NE:%.*]] "test_ne" +; CHECK-COMPAT-DAG: OpName [[EQ:%.*]] "test_eq" +; CHECK-COMPAT-DAG: OpName [[NE:%.*]] "test_ne" ; CHECK-DAG: OpName [[ULT:%.*]] "test_ult" ; CHECK-DAG: OpName [[SLT:%.*]] "test_slt" ; CHECK-DAG: OpName [[ULE:%.*]] "test_ule" @@ -19,6 +25,9 @@ ; CHECK-NEXT: [[R:%.*]] = OpPtrEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[EQ]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_eq(i16* %a, i16* %b) { %r = icmp eq i16* %a, %b ret i1 %r @@ -31,6 +40,9 @@ define i1 @test_eq(i16* %a, i16* %b) { ; CHECK-NEXT: [[R:%.*]] = OpPtrNotEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[NE]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrNotEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_ne(i16* %a, i16* %b) { %r = icmp ne i16* %a, %b ret i1 %r -- GitLab From 1d250d9099a9ba8b53add7eb7db6827e8fc0c8fd Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:08 +0100 Subject: [PATCH 026/343] [SPIR-V] Improve type inference in SPIR-V Backend for opaque pointers (#86283) This PR improves type inference in SPIR-V Backend for opaque pointers, accounting or a case when there is a chain of function calls that allows to deduce formal parameter types from actual arguments. The attached test demonstrates the case. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 123 +++++++++++------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 12 +- .../pointers/type-deduce-by-call-chain.ll | 52 ++++++++ 4 files changed, 138 insertions(+), 50 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 458af9229ed7..5828db6669ff 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -92,6 +92,9 @@ class SPIRVEmitIntrinsics void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); void processParamTypes(Function *F, IRBuilder<> &B); + Type *deduceFunParamType(Function *F, unsigned OpIdx); + Type *deduceFunParamType(Function *F, unsigned OpIdx, + std::unordered_set &FVisited); public: static char ID; @@ -169,6 +172,10 @@ static inline void reportFatalOnTokenType(const Instruction *I) { static Type *deduceElementTypeHelper(Value *I, std::unordered_set &Visited, DenseMap &DeducedElTys) { + // allow to pass nullptr as an argument + if (!I) + return nullptr; + // maybe already known auto It = DeducedElTys.find(I); if (It != DeducedElTys.end()) @@ -182,15 +189,20 @@ static Type *deduceElementTypeHelper(Value *I, // fallback value in case when we fail to deduce a type Type *Ty = nullptr; // look for known basic patterns of type inference - if (auto *Ref = dyn_cast(I)) + if (auto *Ref = dyn_cast(I)) { Ty = Ref->getAllocatedType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getResultElementType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getValueType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, DeducedElTys); + } else if (auto *Ref = dyn_cast(I)) { + if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); + isPointerTy(Src) && isPointerTy(Dest)) + Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, DeducedElTys); + } // remember the found relationship if (Ty) @@ -795,61 +807,80 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, } } -void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { - DenseMap Args; - unsigned i = 0; - for (Argument &Arg : F->args()) { - if (isUntypedPointerTy(Arg.getType()) && - DeducedElTys.find(&Arg) == DeducedElTys.end() && - !HasPointeeTypeAttr(&Arg)) - Args[i] = &Arg; - i++; - } - if (Args.size() == 0) - return; +Type *SPIRVEmitIntrinsics::deduceFunParamType(Function *F, unsigned OpIdx) { + std::unordered_set FVisited; + return deduceFunParamType(F, OpIdx, FVisited); +} + +Type *SPIRVEmitIntrinsics::deduceFunParamType( + Function *F, unsigned OpIdx, std::unordered_set &FVisited) { + // maybe a cycle + if (FVisited.find(F) != FVisited.end()) + return nullptr; + FVisited.insert(F); - // Args contains opaque pointers without element type definition - B.SetInsertPointPastAllocas(F); std::unordered_set Visited; + SmallVector> Lookup; + // search in function's call sites for (User *U : F->users()) { CallInst *CI = dyn_cast(U); - if (!CI) + if (!CI || OpIdx >= CI->arg_size()) continue; - for (unsigned OpIdx = 0; OpIdx < CI->arg_size() && Args.size() > 0; - OpIdx++) { - auto It = Args.find(OpIdx); - Argument *Arg = It == Args.end() ? nullptr : It->second; - if (!Arg) - continue; - Value *OpArg = CI->getArgOperand(OpIdx); - if (!isPointerTy(OpArg->getType())) + Value *OpArg = CI->getArgOperand(OpIdx); + if (!isPointerTy(OpArg->getType())) + continue; + // maybe we already know operand's element type + if (auto It = DeducedElTys.find(OpArg); It != DeducedElTys.end()) + return It->second; + // search in actual parameter's users + for (User *OpU : OpArg->users()) { + Instruction *Inst = dyn_cast(OpU); + if (!Inst || Inst == CI) continue; - // maybe we already know the operand's element type - auto DeducedIt = DeducedElTys.find(OpArg); - Type *ElemTy = - DeducedIt == DeducedElTys.end() ? nullptr : DeducedIt->second; - if (!ElemTy) { - for (User *OpU : OpArg->users()) { - if (Instruction *Inst = dyn_cast(OpU)) { - Visited.clear(); - ElemTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys); - if (ElemTy) - break; - } - } + Visited.clear(); + if (Type *Ty = deduceElementTypeHelper(Inst, Visited, DeducedElTys)) + return Ty; + } + // check if it's a formal parameter of the outer function + if (!CI->getParent() || !CI->getParent()->getParent()) + continue; + Function *OuterF = CI->getParent()->getParent(); + if (FVisited.find(OuterF) != FVisited.end()) + continue; + for (unsigned i = 0; i < OuterF->arg_size(); ++i) { + if (OuterF->getArg(i) == OpArg) { + Lookup.push_back(std::make_pair(OuterF, i)); + break; } - if (ElemTy) { - unsigned AddressSpace = getPointerAddressSpace(Arg->getType()); + } + } + + // search in function parameters + for (auto &Pair : Lookup) { + if (Type *Ty = deduceFunParamType(Pair.first, Pair.second, FVisited)) + return Ty; + } + + return nullptr; +} + +void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { + B.SetInsertPointPastAllocas(F); + DenseMap Args; + for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { + Argument *Arg = F->getArg(OpIdx); + if (isUntypedPointerTy(Arg->getType()) && + DeducedElTys.find(Arg) == DeducedElTys.end() && + !HasPointeeTypeAttr(Arg)) { + if (Type *ElemTy = deduceFunParamType(F, OpIdx)) { CallInst *AssignPtrTyCI = buildIntrWithMD( Intrinsic::spv_assign_ptr_type, {Arg->getType()}, - Constant::getNullValue(ElemTy), Arg, {B.getInt32(AddressSpace)}, B); + Constant::getNullValue(ElemTy), Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); DeducedElTys[AssignPtrTyCI] = ElemTy; DeducedElTys[Arg] = ElemTy; - Args.erase(It); } } - if (Args.size() == 0) - break; } } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 42f8397a3023..ee52163a5d12 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -479,6 +479,7 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( GVar = M->getGlobalVariable(Name); if (GVar == nullptr) { const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type. + // Module takes ownership of the global var. GVar = new GlobalVariable(*M, const_cast(Ty), false, GlobalValue::ExternalLinkage, nullptr, Twine(Name)); diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 5bb8f6084f96..39228e2196b3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -499,6 +499,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); Register GV = I.getOperand(1).getReg(); MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); + (void)II; assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || (*II).getOpcode() == TargetOpcode::COPY || (*II).getOpcode() == SPIRV::OpVariable) && @@ -771,10 +772,13 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); // TODO: check if we have such GV, add init, use buildGlobalVariable. - Type *LLVMArrTy = ArrayType::get( - IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); - GlobalVariable *GV = - new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); + Function &CurFunction = GR.CurMF->getFunction(); + Type *LLVMArrTy = + ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); + // Module takes ownership of the global var. + GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, + true, GlobalValue::InternalLinkage, + Constant::getNullValue(LLVMArrTy)); Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(GV, GR.CurMF, VarReg); diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll new file mode 100644 index 000000000000..703f1e22a032 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll @@ -0,0 +1,52 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[ArgCum:.*]] "_arg_cum" +; CHECK-SPIRV-DAG: OpName %[[FunTest:.*]] "test" +; CHECK-SPIRV-DAG: OpName %[[Addr:.*]] "addr" +; CHECK-SPIRV-DAG: OpName %[[StubObj:.*]] "stub_object" +; CHECK-SPIRV-DAG: OpName %[[MemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooStub:.*]] "foo_stub" +; CHECK-SPIRV-DAG: OpName %[[FooObj:.*]] "foo_object" +; CHECK-SPIRV-DAG: OpName %[[FooMemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooFunc:.*]] "foo" +; CHECK-SPIRV-DAG: %[[TyLong:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[TyVoid:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[TyPtrLong:.*]] = OpTypePointer CrossWorkgroup %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunPtrLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrLong]] +; CHECK-SPIRV-DAG: %[[TyGenPtrLong:.*]] = OpTypePointer Generic %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunGenPtrLongLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyGenPtrLong]] %[[TyLong]] +; CHECK-SPIRV-DAG: %[[Const3:.*]] = OpConstant %[[TyLong]] 3 +; CHECK-SPIRV: %[[FunTest]] = OpFunction %[[TyVoid]] None %[[TyFunPtrLong]] +; CHECK-SPIRV: %[[ArgCum]] = OpFunctionParameter %[[TyPtrLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooFunc]] %[[Addr]] %[[Const3]] +; CHECK-SPIRV: %[[FooStub]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[StubObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[MemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: %[[FooFunc]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[FooObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[FooMemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooStub]] %[[FooObj]] %[[FooMemOrder]] + +define spir_kernel void @test(ptr addrspace(1) noundef align 4 %_arg_cum) { +entry: + %lptr = getelementptr inbounds i32, ptr addrspace(1) %_arg_cum, i64 1 + %addr = addrspacecast ptr addrspace(1) %lptr to ptr addrspace(4) + %object = bitcast ptr addrspace(4) %addr to ptr addrspace(4) + call spir_func void @foo(ptr addrspace(4) %object, i32 3) + ret void +} + +define void @foo_stub(ptr addrspace(4) noundef %stub_object, i32 noundef %mem_order) { +entry: + %object.addr = alloca ptr addrspace(4) + %object.addr.ascast = addrspacecast ptr %object.addr to ptr addrspace(4) + store ptr addrspace(4) %stub_object, ptr addrspace(4) %object.addr.ascast + ret void +} + +define void @foo(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) { + tail call void @foo_stub(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) + ret void +} + -- GitLab From b0d03ccc0855f2bff39160f25fcde06aae07cace Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:46 +0100 Subject: [PATCH 027/343] [SPIR-V] Fix illegal OpConstantComposite instruction with non-const constituents in SPIR-V Backend (#86352) This PR fixes illegal use of OpConstantComposite with non-constant constituents. The test attached to the PR is able now to satisfy `spirv-val` check. Before the fix SPIR-V Backend produced for the attached test case a pattern like ``` %a = OpVariable %_ptr_CrossWorkgroup_uint CrossWorkgroup %uint_123 %11 = OpConstantComposite %_struct_6 %a %a ``` so that `spirv-val` complained with ``` error: line 25: OpConstantComposite Constituent '10[%a]' is not a constant or undef. %11 = OpConstantComposite %_struct_6 %a %a ``` --- .../Target/SPIRV/SPIRVDuplicatesTracker.cpp | 1 + .../lib/Target/SPIRV/SPIRVDuplicatesTracker.h | 9 ++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 8 ++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 90 +++++++++++++++---- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 1 + .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 1 + .../SPIRV/pointers/struct-opaque-pointers.ll | 2 +- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp index d82fb2df4539..7c32bb1968ef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp @@ -39,6 +39,7 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph( prebuildReg2Entry(GT, Reg2Entry); prebuildReg2Entry(FT, Reg2Entry); prebuildReg2Entry(AT, Reg2Entry); + prebuildReg2Entry(MT, Reg2Entry); prebuildReg2Entry(ST, Reg2Entry); for (auto &Op2E : Reg2Entry) { diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h index 96cc621791e9..2ec3fb35ca04 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h @@ -262,6 +262,7 @@ class SPIRVGeneralDuplicatesTracker { SPIRVDuplicatesTracker GT; SPIRVDuplicatesTracker FT; SPIRVDuplicatesTracker AT; + SPIRVDuplicatesTracker MT; SPIRVDuplicatesTracker ST; // NOTE: using MOs instead of regs to get rid of MF dependency to be able @@ -306,6 +307,10 @@ public: AT.add(Arg, MF, R); } + void add(const MachineInstr *MI, const MachineFunction *MF, Register R) { + MT.add(MI, MF, R); + } + void add(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF, Register R) { ST.add(TD, MF, R); @@ -337,6 +342,10 @@ public: return AT.find(const_cast(Arg), MF); } + Register find(const MachineInstr *MI, const MachineFunction *MF) { + return MT.find(const_cast(MI), MF); + } + Register find(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF) { return ST.find(TD, MF); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index ee52163a5d12..db66ed4f0e0f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -123,6 +123,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVType *ElemType, MachineIRBuilder &MIRBuilder) { auto EleOpc = ElemType->getOpcode(); + (void)EleOpc; assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat || EleOpc == SPIRV::OpTypeBool) && "Invalid vector element type"); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index da480b22a525..ed0f90ff89ce 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -94,6 +94,14 @@ public: DT.add(Arg, MF, R); } + void add(const MachineInstr *MI, MachineFunction *MF, Register R) { + DT.add(MI, MF, R); + } + + Register find(const MachineInstr *MI, MachineFunction *MF) { + return DT.find(MI, MF); + } + Register find(const Constant *C, MachineFunction *MF) { return DT.find(C, MF); } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 39228e2196b3..505b19a4d66e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -231,6 +231,9 @@ private: Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const; + + bool wrapIntoSpecConstantOp(MachineInstr &I, + SmallVector &CompositeArgs) const; }; } // end anonymous namespace @@ -1249,6 +1252,24 @@ static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, return N; } +// Return true if the type represents a constant register +static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || + OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; +} + +// Return true if the virtual register represents a constant +static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { + if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) + return isConstReg(MRI, OpDef); + return false; +} + bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1266,16 +1287,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, // check if we may construct a constant vector Register OpReg = I.getOperand(OpIdx).getReg(); - bool IsConst = false; - if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) { - if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && - OpDef->getOperand(1).isReg()) { - if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) - OpDef = RefDef; - } - IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT || - OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; - } + bool IsConst = isConstReg(MRI, OpReg); if (!IsConst && N < 2) report_fatal_error( @@ -1628,6 +1640,48 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg, return Res.constrainAllUses(TII, TRI, RBI); } +// Maybe wrap a value into OpSpecConstantOp +bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( + MachineInstr &I, SmallVector &CompositeArgs) const { + bool Result = true; + unsigned Lim = I.getNumExplicitOperands(); + for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { + Register OpReg = I.getOperand(i).getReg(); + SPIRVType *OpDefine = MRI->getVRegDef(OpReg); + SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); + if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || + OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { + // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed + // by selectAddrSpaceCast() + CompositeArgs.push_back(OpReg); + continue; + } + MachineFunction *MF = I.getMF(); + Register WrapReg = GR.find(OpDefine, MF); + if (WrapReg.isValid()) { + CompositeArgs.push_back(WrapReg); + continue; + } + // Create a new register for the wrapper + WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); + GR.add(OpDefine, MF, WrapReg); + CompositeArgs.push_back(WrapReg); + // Decorate the wrapper register and generate a new instruction + MRI->setType(WrapReg, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); + MachineBasicBlock &BB = *I.getParent(); + Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) + .addDef(WrapReg) + .addUse(GR.getSPIRVTypeID(OpType)) + .addImm(static_cast(SPIRV::Opcode::Bitcast)) + .addUse(OpReg) + .constrainAllUses(TII, TRI, RBI); + if (!Result) + break; + } + return Result; +} + bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1666,17 +1720,21 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_const_composite: { // If no values are attached, the composite is null constant. bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); - unsigned Opcode = - IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; + // Select a proper instruction. + unsigned Opcode = SPIRV::OpConstantNull; + SmallVector CompositeArgs; + if (!IsNull) { + Opcode = SPIRV::OpConstantComposite; + if (!wrapIntoSpecConstantOp(I, CompositeArgs)) + return false; + } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // skip type MD node we already used when generated assign.type for this if (!IsNull) { - for (unsigned i = I.getNumExplicitDefs() + 1; - i < I.getNumExplicitOperands(); ++i) { - MIB.addUse(I.getOperand(i).getReg()); - } + for (Register OpReg : CompositeArgs) + MIB.addUse(OpReg); } return MIB.constrainAllUses(TII, TRI, RBI); } diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index d547f91ba4a5..1f0d8d8cd43a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -543,6 +543,7 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR, Register Dst = ICMP->getOperand(0).getReg(); MachineOperand &PredOp = ICMP->getOperand(1); const auto CC = static_cast(PredOp.getPredicate()); + (void)CC; assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) && MRI.hasOneUse(Dst) && MRI.hasOneDef(CompareReg)); uint64_t Value = getIConstVal(ICMP->getOperand(3).getReg(), &MRI); diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 8dbbd9049844..ff102e318469 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1611,3 +1611,4 @@ multiclass OpcodeOperand value> { // TODO: implement other mnemonics. defm InBoundsPtrAccessChain : OpcodeOperand<70>; defm PtrCastToGeneric : OpcodeOperand<121>; +defm Bitcast : OpcodeOperand<124>; diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll index d426fc4dfd4e..ce3ab8895a59 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]] -- GitLab From cbbbab349e9c412729c3969008cdcb677cc55790 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 12:41:32 +0000 Subject: [PATCH 028/343] [RemoveDIs] Enable direct-to-bitcode writing by default Follow on from #83251. This patch simply enables the behaviour by default in order to provide an easily revertible capstone. --- llvm/lib/IR/BasicBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index f088c7a2cc4e..ae99267f5ba8 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -39,7 +39,7 @@ cl::opt bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, - cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true)); DbgMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && -- GitLab From 3cb024198f6a028089779c0a3cb4d8e753e87c73 Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Mon, 25 Mar 2024 17:17:36 +0800 Subject: [PATCH 029/343] [RISCV][CostModel] Estimate cost of llvm.vector.reduce.fmaximum/fminimum (#80697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ‘llvm.vector.reduce.fmaximum/fminimum.*’ intrinsics propagate NaNs if any element of the vector is a NaN. Following #79402, the patch adds the cost for NaN check (vmfne + vcpop) --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 +++++++++ .../CostModel/RISCV/reduce-fmaximum.ll | 91 +++++++++++++------ .../CostModel/RISCV/reduce-fminimum.ll | 52 +++++------ 3 files changed, 136 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 8f46fdc2f7ca..f75b3d3caa62 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1001,6 +1001,51 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind); } + if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) { + SmallVector Opcodes; + InstructionCost ExtraCost = 0; + switch (IID) { + case Intrinsic::maximum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DstTy->getScalarSizeInBits(); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + + case Intrinsic::minimum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + } + return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind); + } + // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll index 1618c3833a97..f91f13b2d9ec 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll @@ -6,23 +6,37 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) @@ -32,6 +46,13 @@ define float @reduce_fmaximum_f32(float %arg) { %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ret float undef } declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>) @@ -44,21 +65,33 @@ declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) @@ -67,6 +100,12 @@ define double @reduce_fmaximum_f64(double %arg) { %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ret double undef } declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll index 35b18645b1f2..86b84025ad54 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll @@ -6,23 +6,23 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) @@ -44,21 +44,21 @@ declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -- GitLab From e550022b68fc75d32e44faa59ed8f02036cd8f26 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:26:27 +0000 Subject: [PATCH 030/343] [RemoveDIs] Load into new debug info format by default in llvm-dis (#86276) Directly load all bitcode into the new debug info format in llvm-dis. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-dis/llvm-dis.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp index 8e443318dd7d..49154dc46c57 100644 --- a/llvm/tools/llvm-dis/llvm-dis.cpp +++ b/llvm/tools/llvm-dis/llvm-dis.cpp @@ -82,6 +82,8 @@ static cl::opt PrintThinLTOIndexOnly( extern cl::opt WriteNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + namespace { static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) { @@ -169,6 +171,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&DisCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + LLVMContext Context; Context.setDiagnosticHandler( std::make_unique(argv[0])); -- GitLab From 2ef612050844355906e4b67d892a00bbb58c41d6 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:28:01 +0000 Subject: [PATCH 031/343] [RemoveDIs] Do not load into new debug info format from bitcode by default (#86268) This is NFC right now, as the global default behaviour is also "do not load into the new debug info format by default", but we want to change that soon. Additionally unconditionally convert from the new debug info format into if we've loaded into it (e.g., if the bitcode file loaded was already in the new format). The latter change is needed because verify-uselistorder doesn't yet understand DbgRecords (it doesn't know how to map them). The former change is needed because if we load from an old debug format bitcode file but load directly into the new format _and then convert back to the old mode after_, the use-lists of the debug intrinsic functions (the functions' global value uses) change. --- .../verify-uselistorder/verify-uselistorder.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index d929ae09958a..8b299c394e4e 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,6 +68,8 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); +extern cl::opt LoadBitcodeIntoNewDbgInforFormat; + namespace { struct TempFile { @@ -169,8 +171,7 @@ std::unique_ptr TempFile::readBitcode(LLVMContext &Context) const { // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!ModuleOr.get()->IsNewDbgInfoFormat && - "Unexpectedly in new debug info mode"); + ModuleOr.get()->setIsNewDbgInfoFormat(false); return std::move(ModuleOr.get()); } @@ -182,7 +183,7 @@ std::unique_ptr TempFile::readAssembly(LLVMContext &Context) const { Err.print("verify-uselistorder", errs()); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); return M; } @@ -544,6 +545,10 @@ int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "llvm tool to verify use-list order\n"); + // Do not load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + LLVMContext Context; SMDiagnostic Err; @@ -551,7 +556,7 @@ int main(int argc, char **argv) { std::unique_ptr M = parseIRFile(InputFilename, Err, Context); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); if (!M.get()) { Err.print(argv[0], errs()); -- GitLab From 8263a883342d9925a4a1fd9752efc8deda5840fc Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:29:19 +0000 Subject: [PATCH 032/343] [RemoveDIs] Load into new debug info format by default in llvm-link (#86274) Directly load all bitcode into the new debug info format in llvm-link. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-link/llvm-link.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp index 9e7f2c3ebac4..9049cb5e8580 100644 --- a/llvm/tools/llvm-link/llvm-link.cpp +++ b/llvm/tools/llvm-link/llvm-link.cpp @@ -136,6 +136,8 @@ static cl::opt TryUseNewDbgInfoFormat( extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + static ExitOnError ExitOnErr; // Read the specified bitcode file in and return it. This routine searches the @@ -480,6 +482,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&LinkCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. if (TryUseNewDbgInfoFormat) { -- GitLab From 772e316457ef94759804d9f4da0af70d8d2ca4d4 Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Mon, 25 Mar 2024 09:43:41 +0000 Subject: [PATCH 033/343] [FMV] Allow multi versioning without default declaration. (#85454) This was a limitation which has now been lifted. Please read the thread below for more details: https://github.com/llvm/llvm-project/pull/84405#discussion_r1525583647 Basically it allows to separate versioned implementations across different TUs without having to share private header files which contain the default declaration. The ACLE spec has been updated accordingly to make this explicit: "Each version declaration should be visible at the translation unit in which the corresponding function version resides." https://github.com/ARM-software/acle/pull/310 If a resolver is required (because there is a caller in the TU), then a default declaration is implicitly generated. --- clang/lib/CodeGen/CodeGenModule.cpp | 131 ++++--- clang/lib/Sema/SemaDecl.cpp | 6 +- clang/lib/Sema/SemaOverload.cpp | 44 ++- clang/test/CodeGen/attr-target-version.c | 368 ++++++++++++------ clang/test/CodeGenCXX/attr-target-version.cpp | 62 +-- clang/test/Sema/aarch64-sme-func-attrs.c | 8 +- clang/test/Sema/attr-target-version.c | 8 +- clang/test/SemaCXX/attr-target-version.cpp | 11 +- 8 files changed, 403 insertions(+), 235 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index cb153066b28d..ac81df8cf7ad 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3711,7 +3711,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + !FD->isTargetVersionMultiVersion()) return; StringRef MangledName = getMangledName(GD); @@ -4092,6 +4093,23 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM, return llvm::GlobalValue::WeakODRLinkage; } +static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) { + DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl(); + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + StorageClass SC = FD->getStorageClass(); + DeclarationName Name = FD->getNameInfo().getName(); + + FunctionDecl *NewDecl = + FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(), + FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC); + + NewDecl->setIsMultiVersion(); + NewDecl->addAttr(TargetVersionAttr::CreateImplicit( + NewDecl->getASTContext(), "default", NewDecl->getSourceRange())); + + return NewDecl; +} + void CodeGenModule::emitMultiVersionFunctions() { std::vector MVFuncsToEmit; MultiVersionFuncs.swap(MVFuncsToEmit); @@ -4099,70 +4117,54 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - bool EmitResolver = !FD->isTargetVersionMultiVersion(); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + return cast(Func); + }; + + bool HasDefaultDecl = !FD->isTargetVersionMultiVersion(); + bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion(); SmallVector Options; if (FD->isTargetMultiVersion()) { getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr(); - llvm::SmallVector Feats; + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector Feats; + llvm::Function *Func = createFunction(CurFD); + + if (const auto *TA = CurFD->getAttr()) { TA->getAddedFeatures(Feats); - Options.emplace_back(cast(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr(); - if (CurFD->isUsed() || (TVA->isDefaultVersion() && - CurFD->doesThisDeclarationHaveABody())) - EmitResolver = true; - llvm::SmallVector Feats; + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr()) { + bool HasDefaultDef = TVA->isDefaultVersion() && + CurFD->doesThisDeclarationHaveABody(); + HasDefaultDecl |= TVA->isDefaultVersion(); + ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef); TVA->getFeatures(Feats); - Options.emplace_back(cast(Func), - /*Architecture*/ "", Feats); - } + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else + llvm_unreachable("unexpected MultiVersionKind"); }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) + } else if (const auto *TC = FD->getAttr()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } + llvm::Function *Func = createFunction(FD, I); + StringRef Version = TC->getFeatureStr(I); StringRef Architecture; llvm::SmallVector Feature; @@ -4180,16 +4182,23 @@ void CodeGenModule::emitMultiVersionFunctions() { Feature.push_back(Version); } - Options.emplace_back(cast(Func), Architecture, Feature); + Options.emplace_back(Func, Architecture, Feature); } } else { assert(0 && "Expected a target or target_clones multiversion function"); continue; } - if (!EmitResolver) + if (!ShouldEmitResolver) continue; + if (!HasDefaultDecl) { + FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD); + llvm::Function *Func = createFunction(NewFD); + llvm::SmallVector Feats; + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); @@ -4480,7 +4489,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) + if (FD->isTargetVersionMultiVersion() && !FD->isUsed()) + AddDeferredMultiVersionResolverToEmit(GD); + else if (!IsForDefinition) return GetOrCreateMultiVersionResolver(GD); } } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index aa754d47a0c4..73ea155053d7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -11441,9 +11441,9 @@ static bool CheckMultiVersionFirstFunction(Sema &S, FunctionDecl *FD) { "Function lacks multiversion attribute"); const auto *TA = FD->getAttr(); const auto *TVA = FD->getAttr(); - // Target and target_version only causes MV if it is default, otherwise this - // is a normal function. - if ((TA && !TA->isDefaultVersion()) || (TVA && !TVA->isDefaultVersion())) + // The target attribute only causes MV if this declaration is the default, + // otherwise it is treated as a normal function. + if (TA && !TA->isDefaultVersion()) return false; if ((TA || TVA) && CheckMultiVersionValue(S, FD)) { diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index f6bd85bdc646..51450e486eae 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -6865,6 +6865,32 @@ static bool IsAcceptableNonMemberOperatorCandidate(ASTContext &Context, return false; } +static bool isNonViableMultiVersionOverload(FunctionDecl *FD) { + if (FD->isTargetMultiVersionDefault()) + return false; + + if (!FD->getASTContext().getTargetInfo().getTriple().isAArch64()) + return FD->isTargetMultiVersion(); + + if (!FD->isMultiVersion()) + return false; + + // Among multiple target versions consider either the default, + // or the first non-default in the absence of default version. + unsigned SeenAt = 0; + unsigned I = 0; + bool HasDefault = false; + FD->getASTContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + if (FD == CurFD) + SeenAt = I; + else if (CurFD->isTargetMultiVersionDefault()) + HasDefault = true; + ++I; + }); + return HasDefault || SeenAt != 0; +} + /// AddOverloadCandidate - Adds the given function to the set of /// candidate functions, using the given function call arguments. If /// @p SuppressUserConversions, then don't allow user-defined @@ -6970,11 +6996,7 @@ void Sema::AddOverloadCandidate( } } - if (Function->isMultiVersion() && - ((Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()) || - (Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Function)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; return; @@ -7637,11 +7659,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, return; } - if (Method->isMultiVersion() && - ((Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()) || - (Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Method)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } @@ -8127,11 +8145,7 @@ void Sema::AddConversionCandidate( return; } - if (Conversion->isMultiVersion() && - ((Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()) || - (Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Conversion)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index 25129605e76c..dd4cbbf5a898 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -109,21 +109,47 @@ int unused_with_implicit_default_def(void) { return 1; } int unused_with_implicit_forward_default_def(void) { return 0; } __attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; } -// This should generate a normal function. +// This should generate a target version despite the default not being declared. __attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; } +// These shouldn't generate anything. +int unused_version_declarations(void); +__attribute__((target_version("jscvt"))) int unused_version_declarations(void); +__attribute__((target_version("rdma"))) int unused_version_declarations(void); + +// These should generate the default (mangled) version and the resolver. +int default_def_with_version_decls(void) { return 0; } +__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void); +__attribute__((target_version("rdma"))) int default_def_with_version_decls(void); + +// The following is guarded because in NOFMV we get errors for calling undeclared functions. +#ifdef __HAVE_FUNCTION_MULTI_VERSIONING +// This should generate a default declaration, two target versions and the resolver. +__attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; } +__attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; } + +// This should generate a default declaration and the resolver. +__attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void); +__attribute__((target_version("rdma"))) int used_decl_without_default_decl(void); + +int caller(void) { return used_def_without_default_decl() + used_decl_without_default_decl(); } +#endif + //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @fmv.ifunc = weak_odr alias i32 (), ptr @fmv // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e +// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline -// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def // CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def // CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def +// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls +// CHECK: @used_def_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_def_without_default_decl +// CHECK: @used_decl_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_decl_without_default_decl // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver @@ -131,97 +157,121 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver +// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver +// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver +// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver //. // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 -// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-SAME: () #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata -// CHECK-SAME: () #[[ATTR2:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag -// CHECK-SAME: () #[[ATTR3:[0-9]+]] { +// CHECK-SAME: () #[[ATTR4:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp -// CHECK-SAME: () #[[ATTR4:[0-9]+]] { +// CHECK-SAME: () #[[ATTR5:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v -// CHECK-SAME: () #[[ATTR5:[0-9]+]] { +// CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti -// CHECK-SAME: () #[[ATTR6:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 -// CHECK-SAME: () #[[ATTR7:[0-9]+]] { +// CHECK-SAME: () #[[ATTR8:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb -// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-SAME: () #[[ATTR10:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh -// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd -// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -371,35 +421,49 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d.default +// CHECK-SAME: () #[[ATTR11]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -587,7 +651,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -595,7 +659,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -606,7 +670,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -623,228 +687,268 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR12:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 +// CHECK-SAME: () #[[ATTR12]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm +// CHECK-SAME: () #[[ATTR18:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 -// CHECK-SAME: () #[[ATTR10]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm +// CHECK-SAME: () #[[ATTR18]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_without_default -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@caller +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() +// CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() +// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] +// CHECK-NEXT: ret i32 [[ADD]] +// +// +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_def_without_default_decl.default +// +// +// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl.default // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR19:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR20:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR13]] { +// CHECK-SAME: () #[[ATTR15]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd -// CHECK-SAME: () #[[ATTR3]] { +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 14 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR34:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-SAME: () #[[ATTR35:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR9]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 -// -// // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -887,6 +991,28 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def.default // // +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @default_def_with_version_decls.default +// +// // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { @@ -995,40 +1121,50 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NOFMV-NEXT: entry: // CHECK-NOFMV-NEXT: ret i32 0 // +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 0 +// //. -// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index e06121d1a719..8b7273fe3bb5 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -35,7 +35,7 @@ struct MyClass { int unused_with_implicit_forward_default_def(void); int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void); - // This should generate a normal function. + // This should generate a target version despite the default not being declared. int __attribute__((target_version("rdm"))) unused_without_default(void); }; @@ -75,6 +75,13 @@ int bar() { // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver //. +// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: ret i32 1 +// +// // CHECK-LABEL: @_Z3fooi.default( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 @@ -82,6 +89,11 @@ int bar() { // CHECK-NEXT: ret i32 2 // // +// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 3 +// +// // CHECK-LABEL: @_Z3foov.default( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -189,6 +201,14 @@ int bar() { // CHECK-NEXT: ret i32 1 // // +// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// // CHECK-LABEL: @_Z3barv( // CHECK-NEXT: entry: // CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 @@ -250,26 +270,6 @@ int bar() { // CHECK-NEXT: ret ptr @_Z3foov.default // // -// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: ret i32 1 -// -// -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 -// -// -// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 0 -// -// // CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -312,16 +312,16 @@ int bar() { // CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default // //. -// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } -// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } // CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" } // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } //. diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 47dbeca206a9..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -483,14 +483,16 @@ void just_fine(void) {} __arm_locally_streaming __attribute__((target_version("sme2"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} +// expected-error@-1 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} +// expected-cpp-error@-2 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} __attribute__((target_version("default"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} void fmv_caller() { cannot_work_version(); cannot_work_clones(); just_fine(); - just_fine_locally_streaming(); + incompatible_locally_streaming(); } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index e2940c434c2f..cd5be459456e 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -68,13 +68,15 @@ int __attribute__((target_version(""))) unsup1(void) { return 1; } void __attribute__((target_version("crc32"))) unsup2(void) {} void __attribute__((target_version("default+fp16"))) koo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("default+default+default"))) loo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} //expected-error@+2 {{redefinition of 'redef'}} //expected-note@-2 {{previous definition is here}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} -int __attribute__((target_version("sm4"))) def(void); +int def(void); void __attribute__((target_version("dit"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); @@ -83,7 +85,6 @@ int bar() { // expected-error@+2 {{reference to overloaded function could not be resolved; did you mean to call it?}} // expected-note@-3 {{possible target for call}} ovl++; - // expected-error@+1 {{no matching function for call to 'nodef'}} nodef(); return def(); } @@ -92,8 +93,6 @@ int __attribute__((target_version("sha1"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -// expected-note@+1 {{function multiversioning caused by this declaration}} -int __attribute__((target_version("fcma"))) prot(); int __attribute__((target_version("pmull"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} @@ -104,6 +103,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +// expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } int cargs() { return unspec_args(); } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index 0bd710c4e282..b3385f043590 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -9,7 +9,6 @@ void __attribute__((target_version("rcpc3"))) no_def(void); void __attribute__((target_version("mops"))) no_def(void); void __attribute__((target_version("rdma"))) no_def(void); -// expected-error@+1 {{no matching function for call to 'no_def'}} void foo(void) { no_def(); } constexpr int __attribute__((target_version("sve2"))) diff_const(void) { return 1; } @@ -41,6 +40,7 @@ inline int __attribute__((target_version("sme"))) diff_inline(void) { return 1; int __attribute__((target_version("fp16"))) diff_inline(void) { return 2; } inline int __attribute__((target_version("sme"))) diff_inline1(void) { return 1; } +//expected-error@+1 {{multiversioned function declaration has a different inline specification}} int __attribute__((target_version("default"))) diff_inline1(void) { return 2; } int __attribute__((target_version("fcma"))) diff_type1(void) { return 1; } @@ -59,8 +59,7 @@ int __attribute__((target_version("sve2-sha3"))) diff_type3(void) noexcept(true) template int __attribute__((target_version("default"))) temp(T) { return 1; } template int __attribute__((target_version("simd"))) temp1(T) { return 1; } -// expected-error@+1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} -template int __attribute__((target_version("sha3"))) temp1(T) { return 2; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} extern "C" { int __attribute__((target_version("aes"))) extc(void) { return 1; } @@ -70,17 +69,23 @@ int __attribute__((target_version("lse"))) extc(void) { return 1; } auto __attribute__((target_version("default"))) ret1(void) { return 1; } auto __attribute__((target_version("dpb"))) ret2(void) { return 1; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deduced return types}} auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} __attribute__((target_version("sve-i8mm"))) ~Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support defaulted functions}} Cls &__attribute__((target_version("ssbs"))) operator=(Cls &&) = delete; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deleted functions}} virtual void __attribute__((target_version("default"))) vfunc(); virtual void __attribute__((target_version("sm4"))) vfunc1(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support virtual functions}} }; __attribute__((target_version("sha3"))) void Decl(); -- GitLab From 7434a6b96c68b6835a6de6e98199dcfc35981dd5 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 25 Mar 2024 10:50:08 +0100 Subject: [PATCH 034/343] [TableGen] Fix Linker Errors Fix linker errors after landing fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 --- llvm/utils/TableGen/Basic/CMakeLists.txt | 2 +- llvm/utils/TableGen/CMakeLists.txt | 2 +- llvm/utils/TableGen/Common/CMakeLists.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt index f2927d05c175..5a899e3b7c80 100644 --- a/llvm/utils/TableGen/Basic/CMakeLists.txt +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -8,7 +8,7 @@ set(LLVM_LINK_COMPONENTS TableGen ) -add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL +add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL CodeGenIntrinsics.cpp SDNodeProperties.cpp ) diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 14690329cabf..577aeded4be7 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -75,10 +75,10 @@ add_tablegen(llvm-tblgen LLVM X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp + $ $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt index 491d9bd2949d..0e985a083247 100644 --- a/llvm/utils/TableGen/Common/CMakeLists.txt +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -39,7 +39,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL vt_gen ) set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") -target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) # Users may include its headers as "Common/*.h" target_include_directories(LLVMTableGenCommon -- GitLab From 336bdf1a255571f8d894e8befe4be7e9141f7541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Mon, 25 Mar 2024 11:52:29 +0200 Subject: [PATCH 035/343] [verify-uselistorder] Fix a typo, fix linking This fixes a typo from 2ef612050844355906e4b67d892a00bbb58c41d6, which broke the build with errors like: ld.lld: error: undefined symbol: LoadBitcodeIntoNewDbgInforFormat >>> referenced by verify-uselistorder.cpp >>> tools/verify-uselistorder/CMakeFiles/verify-uselistorder.dir/verify-uselistorder.cpp.o:(main) >>> did you mean: LoadBitcodeIntoNewDbgInfoFormat >>> defined in: lib/libLLVMBitReader.a(BitcodeReader.cpp.o) collect2: error: ld returned 1 exit status --- llvm/tools/verify-uselistorder/verify-uselistorder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index 8b299c394e4e..cb07dede1d13 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,7 +68,7 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); -extern cl::opt LoadBitcodeIntoNewDbgInforFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -546,8 +546,8 @@ int main(int argc, char **argv) { "llvm tool to verify use-list order\n"); // Do not load bitcode into the new debug info format by default. - if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) - LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_FALSE; LLVMContext Context; SMDiagnostic Err; -- GitLab From dbfc38ed6b3f2a9be0b1a86b2a074aad69eb58a6 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Mon, 25 Mar 2024 18:57:53 +0900 Subject: [PATCH 036/343] [mlir][bufferization] Add `BufferOriginAnalysis` (#86461) This commit adds the `BufferOriginAnalysis`, which can be queried to check if two buffer SSA values originate from the same allocation. This new analysis is used in the buffer deallocation pass to fold away or simplify `bufferization.dealloc` ops more aggressively. The `BufferOriginAnalysis` is based on the `BufferViewFlowAnalysis`, which collects buffer SSA value "same buffer" dependencies. E.g., given IR such as: ``` %0 = memref.alloc() %1 = memref.subview %0 %2 = memref.subview %1 ``` The `BufferViewFlowAnalysis` will report the following "reverse" dependencies (`resolveReverse`) for `%2`: {`%2`, `%1`, `%0`}. I.e., all buffer SSA values in the reverse use-def chain that originate from the same allocation as `%2`. The `BufferOriginAnalysis` is built on top of that. It handles only simple cases at the moment and may conservatively return "unknown" around certain IR with branches, memref globals and function arguments. This analysis enables additional simplifications during `-buffer-deallocation-simplification`. In particular, "regular" scf.for loop nests, that yield buffers (or reallocations thereof) in the same order as they appear in the iter_args, are now handled much more efficiently. Such IR patterns are generated by the sparse compiler. --- .../Bufferization/IR/BufferizationOps.td | 1 + .../Transforms/BufferViewFlowAnalysis.h | 36 ++++ .../BufferDeallocationSimplification.cpp | 80 ++++----- .../Transforms/BufferViewFlowAnalysis.cpp | 160 ++++++++++++++++-- .../dealloc-loops.mlir | 86 ++++++++++ .../buffer-deallocation-simplification.mlir | 14 +- 6 files changed, 321 insertions(+), 56 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td index 9dc6afcaab31..4f609ddff9a4 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td @@ -10,6 +10,7 @@ #define BUFFERIZATION_OPS include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td" +include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizationBase.td" include "mlir/Interfaces/DestinationStyleOpInterface.td" diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 9e43265c5dfe..4015231c845d 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -53,6 +53,7 @@ public: /// /// Results in resolve(B) returning {B, C} ValueSetT resolve(Value value) const; + ValueSetT resolveReverse(Value value) const; /// Removes the given values from all alias sets. void remove(const SetVector &aliasValues); @@ -73,11 +74,46 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + ValueMapT reverseDependencies; /// A set of all SSA values that may be terminal buffers. DenseSet terminals; }; +/// An is-same-buffer analysis that checks if two SSA values belong to the same +/// buffer allocation or not. +class BufferOriginAnalysis { +public: + BufferOriginAnalysis(Operation *op); + + /// Return "true" if `v1` and `v2` originate from the same buffer allocation. + /// Return "false" if `v1` and `v2` originate from different allocations. + /// Return "nullopt" if we do not know for sure. + /// + /// Example 1: isSameAllocation(%0, %1) == true + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.subview %0 + /// ``` + /// + /// Example 2: isSameAllocation(%0, %1) == false + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// ``` + /// + /// Example 3: isSameAllocation(%0, %2) == nullopt + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// %2 = arith.select %c, %0, %1 + /// ``` + std::optional isSameAllocation(Value v1, Value v2); + +private: + BufferViewFlowAnalysis analysis; +}; + } // namespace mlir #endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERVIEWFLOWANALYSIS_H diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp index e30779868b47..954485cfede3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Analysis/AliasAnalysis.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" #include "mlir/Dialect/Bufferization/Transforms/Passes.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" @@ -34,6 +34,14 @@ using namespace mlir::bufferization; // Helpers //===----------------------------------------------------------------------===// +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, ValueRange memrefs, ValueRange conditions, @@ -49,14 +57,6 @@ static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, return success(); } -/// Given a memref value, return the "base" value by skipping over all -/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. -static Value getViewBase(Value value) { - while (auto viewLikeOp = value.getDefiningOp()) - value = viewLikeOp.getViewSource(); - return value; -} - /// Return "true" if the given values are guaranteed to be different (and /// non-aliasing) allocations based on the fact that one value is the result /// of an allocation and the other value is a block argument of a parent block. @@ -80,12 +80,14 @@ static bool distinctAllocAndBlockArgument(Value v1, Value v2) { /// Checks if `memref` may potentially alias a MemRef in `otherList`. It is /// often a requirement of optimization patterns that there cannot be any /// aliasing memref in order to perform the desired simplification. -static bool potentiallyAliasesMemref(AliasAnalysis &analysis, +static bool potentiallyAliasesMemref(BufferOriginAnalysis &analysis, ValueRange otherList, Value memref) { for (auto other : otherList) { if (distinctAllocAndBlockArgument(other, memref)) continue; - if (!analysis.alias(other, memref).isNo()) + std::optional analysisResult = + analysis.isSameAllocation(other, memref); + if (!analysisResult.has_value() || analysisResult == true) return true; } return false; @@ -129,8 +131,8 @@ namespace { struct RemoveDeallocMemrefsContainedInRetained : public OpRewritePattern { RemoveDeallocMemrefsContainedInRetained(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} /// The passed 'memref' must not have a may-alias relation to any retained /// memref, and at least one must-alias relation. If there is no must-aliasing @@ -147,10 +149,11 @@ struct RemoveDeallocMemrefsContainedInRetained // deallocated in some situations and can thus not be dropped). bool atLeastOneMustAlias = false; for (Value retained : deallocOp.getRetained()) { - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMay()) + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (!analysisResult.has_value()) return failure(); - if (analysisResult.isMust() || analysisResult.isPartial()) + if (analysisResult == true) atLeastOneMustAlias = true; } if (!atLeastOneMustAlias) @@ -161,8 +164,9 @@ struct RemoveDeallocMemrefsContainedInRetained // we can remove that operand later on. for (auto [i, retained] : llvm::enumerate(deallocOp.getRetained())) { Value updatedCondition = deallocOp.getUpdatedConditions()[i]; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { auto disjunction = rewriter.create( deallocOp.getLoc(), updatedCondition, cond); rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(), @@ -206,7 +210,7 @@ struct RemoveDeallocMemrefsContainedInRetained } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Remove memrefs from the `retained` list which are guaranteed to not alias @@ -228,15 +232,15 @@ private: struct RemoveRetainedMemrefsGuaranteedToNotAlias : public OpRewritePattern { RemoveRetainedMemrefsGuaranteedToNotAlias(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { SmallVector newRetainedMemrefs, replacements; for (auto retainedMemref : deallocOp.getRetained()) { - if (potentiallyAliasesMemref(aliasAnalysis, deallocOp.getMemrefs(), + if (potentiallyAliasesMemref(analysis, deallocOp.getMemrefs(), retainedMemref)) { newRetainedMemrefs.push_back(retainedMemref); replacements.push_back({}); @@ -264,7 +268,7 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Split off memrefs to separate dealloc operations to reduce the number of @@ -297,8 +301,8 @@ private: struct SplitDeallocWhenNotAliasingAnyOther : public OpRewritePattern { SplitDeallocWhenNotAliasingAnyOther(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -314,7 +318,7 @@ struct SplitDeallocWhenNotAliasingAnyOther SmallVector otherMemrefs(deallocOp.getMemrefs()); otherMemrefs.erase(otherMemrefs.begin() + i); // Check if `memref` can split off into a separate bufferization.dealloc. - if (potentiallyAliasesMemref(aliasAnalysis, otherMemrefs, memref)) { + if (potentiallyAliasesMemref(analysis, otherMemrefs, memref)) { // `memref` alias with other memrefs, do not split off. remainingMemrefs.push_back(memref); remainingConditions.push_back(cond); @@ -352,7 +356,7 @@ struct SplitDeallocWhenNotAliasingAnyOther } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Check for every retained memref if a must-aliasing memref exists in the @@ -381,8 +385,8 @@ private: struct RetainedMemrefAliasingAlwaysDeallocatedMemref : public OpRewritePattern { RetainedMemrefAliasingAlwaysDeallocatedMemref(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -396,8 +400,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!matchPattern(cond, m_One())) continue; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -411,10 +416,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!extractOp) continue; - AliasResult extractAnalysisResult = - aliasAnalysis.alias(retained, extractOp.getOperand()); - if (extractAnalysisResult.isMust() || - extractAnalysisResult.isPartial()) { + std::optional extractAnalysisResult = + analysis.isSameAllocation(retained, extractOp.getOperand()); + if (extractAnalysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -434,7 +438,7 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; } // namespace @@ -452,13 +456,13 @@ struct BufferDeallocationSimplificationPass : public bufferization::impl::BufferDeallocationSimplificationBase< BufferDeallocationSimplificationPass> { void runOnOperation() override { - AliasAnalysis &aliasAnalysis = getAnalysis(); + BufferOriginAnalysis analysis(getOperation()); RewritePatternSet patterns(&getContext()); patterns.add(&getContext(), - aliasAnalysis); + analysis); populateDeallocOpCanonicalizationPatterns(patterns, &getContext()); if (failed( diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 9a36057425f3..72f47b8b468e 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -19,22 +19,23 @@ using namespace mlir; using namespace mlir::bufferization; +//===----------------------------------------------------------------------===// +// BufferViewFlowAnalysis +//===----------------------------------------------------------------------===// + /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } -/// Find all immediate and indirect dependent buffers this value could -/// potentially have. Note that the resulting set will also contain the value -/// provided as it is a dependent alias of itself. -BufferViewFlowAnalysis::ValueSetT -BufferViewFlowAnalysis::resolve(Value rootValue) const { - ValueSetT result; +static BufferViewFlowAnalysis::ValueSetT +resolveValues(const BufferViewFlowAnalysis::ValueMapT &map, Value value) { + BufferViewFlowAnalysis::ValueSetT result; SmallVector queue; - queue.push_back(rootValue); + queue.push_back(value); while (!queue.empty()) { Value currentValue = queue.pop_back_val(); if (result.insert(currentValue).second) { - auto it = dependencies.find(currentValue); - if (it != dependencies.end()) { + auto it = map.find(currentValue); + if (it != map.end()) { for (Value aliasValue : it->second) queue.push_back(aliasValue); } @@ -43,6 +44,19 @@ BufferViewFlowAnalysis::resolve(Value rootValue) const { return result; } +/// Find all immediate and indirect dependent buffers this value could +/// potentially have. Note that the resulting set will also contain the value +/// provided as it is a dependent alias of itself. +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolve(Value rootValue) const { + return resolveValues(dependencies, rootValue); +} + +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolveReverse(Value rootValue) const { + return resolveValues(reverseDependencies, rootValue); +} + /// Removes the given values from all alias sets. void BufferViewFlowAnalysis::remove(const SetVector &aliasValues) { for (auto &entry : dependencies) @@ -69,8 +83,10 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip_equal(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) { this->dependencies[value].insert(dep); + this->reverseDependencies[dep].insert(value); + } }; // Mark all buffer results and buffer region entry block arguments of the @@ -188,3 +204,127 @@ bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { assert(isa(value.getType()) && "expected memref"); return terminals.contains(value); } + +//===----------------------------------------------------------------------===// +// BufferOriginAnalysis +//===----------------------------------------------------------------------===// + +/// Return "true" if the given value is the result of a memory allocation. +static bool hasAllocateSideEffect(Value v) { + Operation *op = v.getDefiningOp(); + if (!op) + return false; + return hasEffect(op, v); +} + +/// Return "true" if the given value is a function block argument. +static bool isFunctionArgument(Value v) { + auto bbArg = dyn_cast(v); + if (!bbArg) + return false; + Block *b = bbArg.getOwner(); + auto funcOp = dyn_cast(b->getParentOp()); + if (!funcOp) + return false; + return bbArg.getOwner() == &funcOp.getFunctionBody().front(); +} + +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + +BufferOriginAnalysis::BufferOriginAnalysis(Operation *op) : analysis(op) {} + +std::optional BufferOriginAnalysis::isSameAllocation(Value v1, Value v2) { + assert(isa(v1.getType()) && "expected buffer"); + assert(isa(v2.getType()) && "expected buffer"); + + // Skip over all view-like ops. + v1 = getViewBase(v1); + v2 = getViewBase(v2); + + // Fast path: If both buffers are the same SSA value, we can be sure that + // they originate from the same allocation. + if (v1 == v2) + return true; + + // Compute the SSA values from which the buffers `v1` and `v2` originate. + SmallPtrSet origin1 = analysis.resolveReverse(v1); + SmallPtrSet origin2 = analysis.resolveReverse(v2); + + // Originating buffers are "terminal" if they could not be traced back any + // further by the `BufferViewFlowAnalysis`. Examples of terminal buffers: + // - function block arguments + // - values defined by allocation ops such as "memref.alloc" + // - values defined by ops that are unknown to the buffer view flow analysis + // - values that are marked as "terminal" in the `BufferViewFlowOpInterface` + SmallPtrSet terminal1, terminal2; + + // While gathering terminal buffers, keep track of whether all terminal + // buffers are newly allocated buffer or function entry arguments. + bool allAllocs1 = true, allAllocs2 = true; + bool allAllocsOrFuncEntryArgs1 = true, allAllocsOrFuncEntryArgs2 = true; + + // Helper function that gathers terminal buffers among `origin`. + auto gatherTerminalBuffers = [this](const SmallPtrSet &origin, + SmallPtrSet &terminal, + bool &allAllocs, + bool &allAllocsOrFuncEntryArgs) { + for (Value v : origin) { + if (isa(v.getType()) && analysis.mayBeTerminalBuffer(v)) { + terminal.insert(v); + allAllocs &= hasAllocateSideEffect(v); + allAllocsOrFuncEntryArgs &= + isFunctionArgument(v) || hasAllocateSideEffect(v); + } + } + assert(!terminal.empty() && "expected non-empty terminal set"); + }; + + // Gather terminal buffers for `v1` and `v2`. + gatherTerminalBuffers(origin1, terminal1, allAllocs1, + allAllocsOrFuncEntryArgs1); + gatherTerminalBuffers(origin2, terminal2, allAllocs2, + allAllocsOrFuncEntryArgs2); + + // If both `v1` and `v2` have a single matching terminal buffer, they are + // guaranteed to originate from the same buffer allocation. + if (llvm::hasSingleElement(terminal1) && llvm::hasSingleElement(terminal2) && + *terminal1.begin() == *terminal2.begin()) + return true; + + // At least one of the two values has multiple terminals. + + // Check if there is overlap between the terminal buffers of `v1` and `v2`. + bool distinctTerminalSets = true; + for (Value v : terminal1) + distinctTerminalSets &= !terminal2.contains(v); + // If there is overlap between the terminal buffers of `v1` and `v2`, we + // cannot make an accurate decision without further analysis. + if (!distinctTerminalSets) + return std::nullopt; + + // If `v1` originates from only allocs, and `v2` is guaranteed to originate + // from different allocations (that is guaranteed if `v2` originates from + // only distinct allocs or function entry arguments), we can be sure that + // `v1` and `v2` originate from different allocations. The same argument can + // be made when swapping `v1` and `v2`. + bool isolatedAlloc1 = allAllocs1 && (allAllocs2 || allAllocsOrFuncEntryArgs2); + bool isolatedAlloc2 = (allAllocs1 || allAllocsOrFuncEntryArgs1) && allAllocs2; + if (isolatedAlloc1 || isolatedAlloc2) + return false; + + // Otherwise: We do not know whether `v1` and `v2` originate from the same + // allocation or not. + // TODO: Function arguments are currently handled conservatively. We assume + // that they could be the same allocation. + // TODO: Terminals other than allocations and function arguments are + // currently handled conservatively. We assume that they could be the same + // allocation. E.g., we currently return "nullopt" for values that originate + // from different "memref.get_global" ops (with different symbols). + return std::nullopt; +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir new file mode 100644 index 000000000000..53b28c3aab6f --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir @@ -0,0 +1,86 @@ +// RUN: mlir-opt %s -expand-realloc="emit-deallocs=false" -ownership-based-buffer-deallocation="private-function-dynamic-ownership=true" -canonicalize -buffer-deallocation-simplification | FileCheck %s + +// A function that reallocates two buffer inside of a loop. The simplification +// pass should be able to figure out that the iter_args are always originating +// from different allocations. IR like this one appears in the sparse compiler. + +// CHECK-LABEL: func private @loop_with_realloc( +func.func private @loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + // CHECK: %[[m2:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m2 = memref.realloc %arg0(%s2) : memref to memref + // CHECK: %[[m3:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m3 = memref.realloc %arg1(%s2) : memref to memref + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg0]] + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg1]] + // CHECK: %[[d0:.*]] = bufferization.dealloc (%[[base0]] : memref) if (%[[o0]]) retain (%[[m2]]#0 : memref) + // CHECK: %[[d1:.*]] = bufferization.dealloc (%[[base1]] : memref) if (%[[o1]]) retain (%[[m3]]#0 : memref) + // CHECK-DAG: %[[o2:.*]] = arith.ori %[[d0]], %[[m2]]#1 + // CHECK-DAG: %[[o3:.*]] = arith.ori %[[d1]], %[[m3]]#1 + // CHECK: scf.yield %[[m2]]#0, %[[m3]]#0, %[[o2]], %[[o3]] + scf.yield %m2, %m3 : memref, memref + } + + // CHECK: %[[d2:.*]] = bufferization.dealloc (%[[m0]] : memref) if (%[[true]]) retain (%[[r]]#0 : memref) + // CHECK: %[[d3:.*]] = bufferization.dealloc (%[[m1]] : memref) if (%[[true]]) retain (%[[r]]#1 : memref) + // CHECK-DAG: %[[or0:.*]] = arith.ori %[[d2]], %[[r]]#2 + // CHECK-DAG: %[[or1:.*]] = arith.ori %[[d3]], %[[r]]#3 + // CHECK: return %[[r]]#0, %[[r]]#1, %[[or0]], %[[or1]] + return %r0, %r1 : memref, memref +} + +// ----- + +// The yielded values of the loop are swapped. Therefore, the +// bufferization.dealloc before the func.return can now longer be split, +// because %r0 could originate from either %m0 and %m1 (same for %r1). + +// CHECK-LABEL: func private @swapping_loop_with_realloc( +func.func private @swapping_loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + %m2 = memref.realloc %arg0(%s2) : memref to memref + %m3 = memref.realloc %arg1(%s2) : memref to memref + scf.yield %m3, %m2 : memref, memref + } + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#0 + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#1 + // CHECK: %[[d:.*]]:2 = bufferization.dealloc (%[[m0]], %[[m1]], %[[base0]], %[[base1]] : {{.*}}) if (%[[true]], %[[true]], %[[r]]#2, %[[r]]#3) retain (%[[r]]#0, %[[r]]#1 : {{.*}}) + // CHECK: return %[[r]]#0, %[[r]]#1, %[[d]]#0, %[[d]]#1 + return %r0, %r1 : memref, memref +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir index eee69acbe821..b40a17cf800b 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir @@ -92,15 +92,13 @@ func.func @dealloc_split_when_no_other_aliasing(%arg0: i1, %arg1: memref<2xi32>, // CHECK-NEXT: [[ALLOC0:%.+]] = memref.alloc( // CHECK-NEXT: [[ALLOC1:%.+]] = memref.alloc( // CHECK-NEXT: [[V0:%.+]] = arith.select{{.*}}[[ALLOC0]], [[ALLOC1]] : -// COM: there is only one value in the retained list because the -// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here and -// COM: removes %arg1 from the list. In the second dealloc, this does not apply -// COM: because function arguments are assumed potentially alias (even if the -// COM: types don't exactly match). +// COM: there is only one value in the retained lists because the +// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here: +// COM: - %alloc is guaranteed to not alias with %arg1. +// COM: - %arg2 is guaranteed to not alias with %0. // CHECK-NEXT: [[V1:%.+]] = bufferization.dealloc ([[ALLOC0]] : memref<2xi32>) if ([[ARG0]]) retain ([[V0]] : memref<2xi32>) -// CHECK-NEXT: [[V2:%.+]]:2 = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]], [[V0]] : memref<2xi32>, memref<2xi32>) -// CHECK-NEXT: [[V3:%.+]] = arith.ori [[V1]], [[V2]]#1 -// CHECK-NEXT: return [[V2]]#0, [[V3]] : +// CHECK-NEXT: [[V2:%.+]] = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]] : memref<2xi32>) +// CHECK-NEXT: return [[V2]], [[V1]] : // ----- -- GitLab From 94a550dab26c4b30a187dd8e3ce431e0f915923b Mon Sep 17 00:00:00 2001 From: Mariusz Sikora Date: Mon, 25 Mar 2024 11:00:59 +0100 Subject: [PATCH 037/343] [AMDGPU][NFC] Rename Feature GFX11FullVGPRs to 1_5xVGPRs (#86468) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 14 ++++++++------ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index c877658cd38e..37dcfef3b2a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -831,10 +831,12 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard", "Does not need SW waitstates" >; -def FeatureGFX11FullVGPRs : SubtargetFeature<"gfx11-full-vgprs", - "HasGFX11FullVGPRs", +// Allocate 1536 VGPRs for wave32 and 768 VGPRs for wave64 +// with allocation granularity 24 for wave32 and 12 for wave64 +def Feature1_5xVGPRs : SubtargetFeature<"allocate1_5xvgprs", + "Has1_5xVGPRs", "true", - "GFX11 with 50% more physical VGPRs and 50% larger allocation granule than GFX10" + "Has 50% more physical VGPRs and 50% larger allocation granule" >; @@ -1491,12 +1493,12 @@ def FeatureISAVersion11_0_Common : FeatureSet< def FeatureISAVersion11_0_0 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs, + [Feature1_5xVGPRs, FeatureUserSGPRInit16Bug])>; def FeatureISAVersion11_0_1 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs])>; + [Feature1_5xVGPRs])>; def FeatureISAVersion11_0_2 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, @@ -1517,7 +1519,7 @@ def FeatureISAVersion11_5_1 : FeatureSet< [FeatureSALUFloatInsts, FeatureDPPSrc1SGPR, FeatureVGPRSingleUseHintInsts, - FeatureGFX11FullVGPRs])>; + Feature1_5xVGPRs])>; def FeatureISAVersion12 : FeatureSet< [FeatureGFX12, diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index ca51da659c33..4da10beabe31 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -223,7 +223,7 @@ protected: bool HasImageStoreD16Bug = false; bool HasImageGather4D16Bug = false; bool HasMSAALoadDstSelBug = false; - bool HasGFX11FullVGPRs = false; + bool Has1_5xVGPRs = false; bool HasMADIntraFwdBug = false; bool HasVOPDInsts = false; bool HasVALUTransUseHazard = false; @@ -1202,7 +1202,7 @@ public: /// target. bool hasNullExportTarget() const { return !GFX11Insts; } - bool hasGFX11FullVGPRs() const { return HasGFX11FullVGPRs; } + bool has1_5xVGPRs() const { return Has1_5xVGPRs; } bool hasVOPDInsts() const { return HasVOPDInsts; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index a90dc32d396f..7bb84d78442b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1087,7 +1087,7 @@ unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, *EnableWavefrontSize32 : STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 24 : 12; if (hasGFX10_3Insts(*STI)) @@ -1114,7 +1114,7 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { if (!isGFX10Plus(*STI)) return 256; bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 1536 : 768; return IsWave32 ? 1024 : 512; } -- GitLab From 37785fedabd8fa752129ef5bac3462311af91c35 Mon Sep 17 00:00:00 2001 From: Discookie Date: Mon, 25 Mar 2024 10:08:56 +0000 Subject: [PATCH 038/343] [clang][analyzer] Bring cplusplus.ArrayDelete out of alpha (#83985) The checker finds a type of undefined behavior, where if the type of a pointer to an object-array is different from the objects' underlying type, calling `delete[]` is undefined, as the size of the two objects might be different. The checker has been in alpha for a while now, it is a simple checker that causes no crashes, and considering the severity of the issue, it has a low result-count on open-source projects (in my last test-run on my usual projects, it had 0 results). This commit cleans up the documentation and adds docs for the limitation related to tracking through references, in addition to moving it to `cplusplus`. --------- Co-authored-by: Balazs Benics Co-authored-by: whisperity --- clang/docs/analyzer/checkers.rst | 69 ++++++++++++------- .../clang/StaticAnalyzer/Checkers/Checkers.td | 10 +-- .../Checkers/CXXDeleteChecker.cpp | 4 +- clang/test/Analysis/ArrayDelete.cpp | 2 +- clang/www/analyzer/alpha_checks.html | 20 ------ clang/www/analyzer/available_checks.html | 27 ++++++++ 6 files changed, 80 insertions(+), 52 deletions(-) diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst index fe2115149142..66da1c7b35f2 100644 --- a/clang/docs/analyzer/checkers.rst +++ b/clang/docs/analyzer/checkers.rst @@ -340,6 +340,51 @@ cplusplus C++ Checkers. +.. _cplusplus-ArrayDelete: + +cplusplus.ArrayDelete (C++) +""""""""""""""""""""""""""" + +Reports destructions of arrays of polymorphic objects that are destructed as +their base class. If the dynamic type of the array is different from its static +type, calling `delete[]` is undefined. + +This checker corresponds to the SEI CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. + +.. code-block:: cpp + + class Base { + public: + virtual ~Base() {} + }; + class Derived : public Base {}; + + Base *create() { + Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here + return x; + } + + void foo() { + Base *x = create(); + delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + +**Limitations** + +The checker does not emit note tags when casting to and from reference types, +even though the pointer values are tracked across references. + +.. code-block:: cpp + + void foo() { + Derived *d = new Derived[10]; + Derived &dref = *d; + + Base &bref = static_cast(dref); // no note + Base *b = &bref; + delete[] b; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + .. _cplusplus-InnerPointer: cplusplus.InnerPointer (C++) @@ -2139,30 +2184,6 @@ Either the comparison is useless or there is division by zero. alpha.cplusplus ^^^^^^^^^^^^^^^ -.. _alpha-cplusplus-ArrayDelete: - -alpha.cplusplus.ArrayDelete (C++) -""""""""""""""""""""""""""""""""" -Reports destructions of arrays of polymorphic objects that are destructed as their base class. -This checker corresponds to the CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. - -.. code-block:: cpp - - class Base { - virtual ~Base() {} - }; - class Derived : public Base {} - - Base *create() { - Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here - return x; - } - - void foo() { - Base *x = create(); - delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined - } - .. _alpha-cplusplus-DeleteWithNonVirtualDtor: alpha.cplusplus.DeleteWithNonVirtualDtor (C++) diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td index 686e5e99f4a6..bf46766d44b3 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td @@ -622,6 +622,11 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">, let ParentPackage = Cplusplus in { +def ArrayDeleteChecker : Checker<"ArrayDelete">, + HelpText<"Reports destructions of arrays of polymorphic objects that are " + "destructed as their base class.">, + Documentation; + def InnerPointerChecker : Checker<"InnerPointer">, HelpText<"Check for inner pointers of C++ containers used after " "re/deallocation">, @@ -777,11 +782,6 @@ def ContainerModeling : Checker<"ContainerModeling">, Documentation, Hidden; -def CXXArrayDeleteChecker : Checker<"ArrayDelete">, - HelpText<"Reports destructions of arrays of polymorphic objects that are " - "destructed as their base class.">, - Documentation; - def DeleteWithNonVirtualDtorChecker : Checker<"DeleteWithNonVirtualDtor">, HelpText<"Reports destructions of polymorphic objects with a non-virtual " "destructor in their base class">, diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp index b4dee1e300e8..1b1226a7f1a7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp @@ -220,11 +220,11 @@ CXXDeleteChecker::PtrCastVisitor::VisitNode(const ExplodedNode *N, /*addPosRange=*/true); } -void ento::registerCXXArrayDeleteChecker(CheckerManager &mgr) { +void ento::registerArrayDeleteChecker(CheckerManager &mgr) { mgr.registerChecker(); } -bool ento::shouldRegisterCXXArrayDeleteChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterArrayDeleteChecker(const CheckerManager &mgr) { return true; } diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp index 3b8d49552376..6887e0a35fb8 100644 --- a/clang/test/Analysis/ArrayDelete.cpp +++ b/clang/test/Analysis/ArrayDelete.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s +// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s struct Base { virtual ~Base() = default; diff --git a/clang/www/analyzer/alpha_checks.html b/clang/www/analyzer/alpha_checks.html index 7bbe4a20288f..f040d1957b0f 100644 --- a/clang/www/analyzer/alpha_checks.html +++ b/clang/www/analyzer/alpha_checks.html @@ -307,26 +307,6 @@ void test(int x) { - - - -