From 20e0bacd0560382a31ad0d4ecc7472bd4a99c659 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 19:08:47 -0500 Subject: [PATCH 001/879] [Libomptarget][Fix] Remove duplicate version script for host builds Summary: This causes an error on some linkers and was mistakenly kept in. --- openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index d30680e10431..ccbf7d033fd6 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -31,11 +31,6 @@ else() target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) endif() -if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.${machine} PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") -endif() - # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.${machine} LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") -- GitLab From dc43aa439ecee90a9f51c6c1e46df8be61e0457c Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Fri, 22 Mar 2024 17:38:34 -0700 Subject: [PATCH 002/879] [asan][windows] When compiling with clang-cl or MSVC pass /Zl (that's a little L) (#85874) /Zl is the equivalent of -nodefaultlibs. The idea here is to make sure that the asan runtime doesn't have any defaultlibs directives, which makes it easier to mix an asan runtime built with the dynamic CRT with an application built with the static CRT (or vise-versa). This is part of the overall effort to remove the static asan runtime on windows entirely: https://github.com/llvm/llvm-project/pull/81677 Co-authored-by: Amy Wishnousky --- compiler-rt/lib/asan/CMakeLists.txt | 3 +++ compiler-rt/lib/sanitizer_common/CMakeLists.txt | 2 ++ compiler-rt/lib/ubsan/CMakeLists.txt | 1 + 3 files changed, 6 insertions(+) diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt index f83ae82d4293..601750f72175 100644 --- a/compiler-rt/lib/asan/CMakeLists.txt +++ b/compiler-rt/lib/asan/CMakeLists.txt @@ -85,6 +85,9 @@ SET(ASAN_HEADERS include_directories(..) set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) + +append_list_if(MSVC /Zl ASAN_CFLAGS) + set(ASAN_COMMON_DEFINITIONS ${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION}) append_rtti_flag(OFF ASAN_CFLAGS) diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt index f762524c333a..f2b4ac72ae15 100644 --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -218,6 +218,8 @@ include_directories(..) set(SANITIZER_COMMON_DEFINITIONS HAVE_RPC_XDR_H=${HAVE_RPC_XDR_H}) +# note: L not I, this is nodefaultlibs for msvc +append_list_if(MSVC /Zl SANITIZER_COMMON_CFLAGS) set(SANITIZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) # Too many existing bugs, needs cleanup. diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt index 3f1e12ed9ac6..db0b33f1276e 100644 --- a/compiler-rt/lib/ubsan/CMakeLists.txt +++ b/compiler-rt/lib/ubsan/CMakeLists.txt @@ -41,6 +41,7 @@ set(UBSAN_HEADERS include_directories(..) set(UBSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +append_list_if(MSVC /Zl UBSAN_CFLAGS) append_rtti_flag(OFF UBSAN_CFLAGS) append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CFLAGS) -- GitLab From b723c57f8fdb12ca8eaa6c5ee2afb820aaeb56c5 Mon Sep 17 00:00:00 2001 From: Corentin Jabot Date: Sat, 23 Mar 2024 09:28:33 +0900 Subject: [PATCH 003/879] [Clang] Update the C++ page with papers approved in Tokyo --- clang/www/cxx_status.html | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 1e36b90356c3..c1d95dadbb27 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -163,7 +163,47 @@ C++23, informally referred to as C++26.

P2864R2 Clang 18 - + + + Disallow Binding a Returned Glvalue to a Temporary + P2748R5 + No + + + Clarifying rules for brace elision in aggregate initialization + P3106R1 (DR) + No + + + Attributes for Structured Bindings + P0609R3 + No + + + Module Declarations Shouldn’t be Macros + P3034R1 (DR) + No + + + Trivial infinite loops are not Undefined Behavior + P2809R3 (DR) + No + + + Erroneous behaviour for uninitialized reads + P2795R5 + No + + + = delete("should have a reason"); + P2573R2 + No + + + Variadic friends + P2893R3 + No + -- GitLab From 2f6b1b4b30e3a719b1744baa4cd1ece504998c6e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 18:10:21 -0700 Subject: [PATCH 004/879] [ORC] Add default visibility to required JIT functions (#86322) If you build LLVM with `-DCMAKE_CXX_VISIBILITY_PRESET=hidden` to help reduce binary size, these symbols end up becoming local, and getting stripped. This forces default visibility to override the global setting in that case. Relevant: https://github.com/llvm/llvm-project/issues/62815#issuecomment-1560078260 --- llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 8a4145a6b02a..7529d9cef67e 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -26,11 +26,13 @@ extern "C" { // We put information about the JITed function in this global, which the // debugger reads. Make sure to specify the version statically, because the // debugger checks the version before we can set it during runtime. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0, nullptr, nullptr}; // Debuggers that implement the GDB JIT interface put a special breakpoint in // this function. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { // The noinline and the asm prevent calls to this function from being // optimized out. -- GitLab From 84e20cb0cbc64bb2d8299b30da3ce8e6421c203e Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 16:56:48 -0700 Subject: [PATCH 005/879] [TableGen] Minor cleanup in CodeGenRegBank::getConcatSubRegIndex. NFC Mark a variable const. Capitalize a variable name. I'm going to add HwMode support to this code and wanted to clean it up a bit beforehand. --- llvm/utils/TableGen/CodeGenRegisters.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp index 40af0d3077b2..e851d4c16bf7 100644 --- a/llvm/utils/TableGen/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -1400,11 +1400,11 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( // None exists, synthesize one. std::string Name = Parts.front()->getName(); // Determine whether all parts are contiguous. - bool isContinuous = true; + bool IsContinuous = true; unsigned Size = Parts.front()->Size; unsigned LastOffset = Parts.front()->Offset; unsigned LastSize = Parts.front()->Size; - unsigned UnknownSize = (uint16_t)-1; + const unsigned UnknownSize = (uint16_t)-1; for (unsigned i = 1, e = Parts.size(); i != e; ++i) { Name += '_'; Name += Parts[i]->getName(); @@ -1413,13 +1413,13 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( else Size += Parts[i]->Size; if (LastSize == UnknownSize || Parts[i]->Offset != (LastOffset + LastSize)) - isContinuous = false; + IsContinuous = false; LastOffset = Parts[i]->Offset; LastSize = Parts[i]->Size; } Idx = createSubRegIndex(Name, Parts.front()->getNamespace()); Idx->Size = Size; - Idx->Offset = isContinuous ? Parts.front()->Offset : -1; + Idx->Offset = IsContinuous ? Parts.front()->Offset : -1; Idx->ConcatenationOf.assign(Parts.begin(), Parts.end()); return Idx; } -- GitLab From 76fdb5902fbadbc08c6742156071431d8ad801ea Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 10:37:53 +0800 Subject: [PATCH 006/879] [NewPM][DirectX] Add DirectXPassRegistry.def NFCI (#86242) Prepare migration for dag-isel --- .../Target/DirectX/DirectXPassRegistry.def | 29 +++++++++++++++++++ .../Target/DirectX/DirectXTargetMachine.cpp | 20 ++----------- 2 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 llvm/lib/Target/DirectX/DirectXPassRegistry.def diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def new file mode 100644 index 000000000000..bbf0c254bb69 --- /dev/null +++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def @@ -0,0 +1,29 @@ +//===- DirectXPassRegistry.def - Registry of DirectX passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// DirectX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_ANALYSIS +#define MODULE_ANALYSIS(NAME, CREATE_PASS) +#endif +MODULE_ANALYSIS("dx-shader-flags", dxil::ShaderFlagsAnalysis()) +MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis()) +#undef MODULE_ANALYSIS + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +// TODO: rename to print after NPM switch +MODULE_PASS("print-dx-shader-flags", DXILResourcePrinterPass(dbgs())) +MODULE_PASS("print-dxil-resource", DXILResourcePrinterPass(dbgs())) +#undef MODULE_PASS diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp index 03c825b3977d..bebca0675522 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp @@ -104,24 +104,8 @@ DirectXTargetMachine::~DirectXTargetMachine() {} void DirectXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "print-dxil-resource") { - PM.addPass(DXILResourcePrinterPass(dbgs())); - return true; - } - if (PassName == "print-dx-shader-flags") { - PM.addPass(dxil::ShaderFlagsAnalysisPrinter(dbgs())); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](ModuleAnalysisManager &MAM) { - MAM.registerPass([&] { return DXILResourceAnalysis(); }); - MAM.registerPass([&] { return dxil::ShaderFlagsAnalysis(); }); - }); +#define GET_PASS_REGISTRY "DirectXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" } bool DirectXTargetMachine::addPassesToEmitFile( -- GitLab From 7ac7d418ac2b16fd44789dcf48e2b5d73de3e715 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 11:20:18 +0800 Subject: [PATCH 007/879] [NewPM][NVPTX] Add NVPTXPassRegistry.def NFCI (#86246) Prepare for dag-isel migration. --- llvm/lib/Target/NVPTX/NVPTXPassRegistry.def | 40 +++++++++++++++++++ llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 41 +------------------- 2 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def diff --git a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def new file mode 100644 index 000000000000..6ff15ab6f13c --- /dev/null +++ b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def @@ -0,0 +1,40 @@ +//===- NVPTXPassRegistry.def - Registry of NVPTX passes ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// NVPTX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("generic-to-nvvm", GenericToNVVMPass()) +MODULE_PASS("nvptx-lower-ctor-dtor", NVPTXCtorDtorLoweringPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("nvptx-aa", NVPTXAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("nvvm-intr-range", NVVMIntrRangePass()) +FUNCTION_PASS("nvvm-reflect", NVVMReflectPass()) +#undef FUNCTION_PASS diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 69d4596f7843..78f48652c992 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -227,45 +227,8 @@ void NVPTXTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { void NVPTXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "nvvm-reflect") { - PM.addPass(NVVMReflectPass()); - return true; - } - if (PassName == "nvvm-intr-range") { - PM.addPass(NVVMIntrRangePass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return NVPTXAA(); }); - }); - - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "nvptx-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); - - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "nvptx-lower-ctor-dtor") { - PM.addPass(NVPTXCtorDtorLoweringPass()); - return true; - } - if (PassName == "generic-to-nvvm") { - PM.addPass(GenericToNVVMPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "NVPTXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { -- GitLab From 2aa5bae0c03f1f857d0ae2a881b223c4a521853f Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 12:53:26 +0800 Subject: [PATCH 008/879] [NewPM][BPF] Add BPFPassRegistry.def NFCI (#86241) Prepare migration for dag-isel. --- llvm/lib/Target/BPF/BPFPassRegistry.def | 32 ++++++++++++++++++++++++ llvm/lib/Target/BPF/BPFTargetMachine.cpp | 25 ++++++------------ 2 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 llvm/lib/Target/BPF/BPFPassRegistry.def diff --git a/llvm/lib/Target/BPF/BPFPassRegistry.def b/llvm/lib/Target/BPF/BPFPassRegistry.def new file mode 100644 index 000000000000..73a8ef2f95bf --- /dev/null +++ b/llvm/lib/Target/BPF/BPFPassRegistry.def @@ -0,0 +1,32 @@ +//===- BPFPassRegistry.def - Registry of BPF passes -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// BPF backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("bpf-aspace-simplify", BPFASpaceCastSimplifyPass()) +FUNCTION_PASS("bpf-ir-peephole", BPFIRPeepholePass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "bpf-preserve-static-offset", "BPFPreserveStaticOffsetPass", + [=](bool AllowPartial) { + return BPFPreserveStaticOffsetPass(AllowPartial); + }, + parseBPFPreserveStaticOffsetOptions, "allow-partial") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 5f26bec2e390..a7bed69b0f2a 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -108,25 +108,16 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) { return new BPFPassConfig(*this, PM); } +static Expected parseBPFPreserveStaticOffsetOptions(StringRef Params) { + return PassBuilder::parseSinglePassOption(Params, "allow-partial", + "BPFPreserveStaticOffsetPass"); +} + void BPFTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &FPM, - ArrayRef) { - if (PassName == "bpf-ir-peephole") { - FPM.addPass(BPFIRPeepholePass()); - return true; - } - if (PassName == "bpf-preserve-static-offset") { - FPM.addPass(BPFPreserveStaticOffsetPass(false)); - return true; - } - if (PassName == "bpf-aspace-simplify") { - FPM.addPass(BPFASpaceCastSimplifyPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "BPFPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerPipelineStartEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { FunctionPassManager FPM; -- GitLab From 635ea257eca7c8e95c6ea30ca3816a0b5584ab37 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 13:06:58 +0800 Subject: [PATCH 009/879] [NewPM] Fix BPF build (#86379) Add Passes in dependency list --- llvm/lib/Target/BPF/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index cb21ed03a86c..eade4cacb710 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -54,6 +54,7 @@ add_llvm_target(BPFCodeGen GlobalISel IPO MC + Passes Scalar SelectionDAG Support -- GitLab From 6c1932ffd82e733325180fe13ef46b24ff606eab Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:57:35 +0800 Subject: [PATCH 010/879] [LLVM] Pass APInt by const reference. NFC. (#86278) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=32d6611af69bf4e76373f9bc7d9649650f760e48&stat=instructions:u --- llvm/include/llvm/Analysis/MemoryBuiltins.h | 2 +- llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +- llvm/include/llvm/MC/MCStreamer.h | 2 +- llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 3 ++- llvm/lib/MC/MCStreamer.cpp | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index 37ce1518f00c..da092866de65 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -217,7 +217,7 @@ struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} - static bool known(APInt V) { return V.getBitWidth() > 1; } + static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; /// Evaluate the size and offset of an object pointed to by a Value* diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 4785e93d72d1..574c63552ce0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -883,7 +883,7 @@ public: /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, Step, Step * 2, Step * 3, ...> - SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); + SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal); /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, 1, 2, 3, ...> diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 671511ab4b88..69867620e1bf 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -740,7 +740,7 @@ public: /// Special case of EmitValue that avoids the client having /// to pass in a MCExpr for constant integers. virtual void emitIntValue(uint64_t Value, unsigned Size); - virtual void emitIntValue(APInt Value); + virtual void emitIntValue(const APInt &Value); /// Special case of EmitValue that avoids the client having to pass /// in a MCExpr for constant integers & prints in Hex format for certain diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 8d608f6ac5e4..1b25da8833e4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -7855,7 +7855,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) { //{ (Src & Mask) >> N } | { (Src << N) & Mask } static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, - MachineInstrBuilder Src, APInt Mask) { + MachineInstrBuilder Src, const APInt &Mask) { const LLT Ty = Dst.getLLTTy(*B.getMRI()); MachineInstrBuilder C_N = B.buildConstant(Ty, N); MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index cd6f083243d0..e2c07e7cb997 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2031,7 +2031,8 @@ SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { return getStepVector(DL, ResVT, One); } -SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { +SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, + const APInt &StepVal) { assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); if (ResVT.isScalableVector()) return getNode( diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index d0395770ae8b..176d55aa890b 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -141,7 +141,7 @@ void MCStreamer::emitIntValue(uint64_t Value, unsigned Size) { unsigned Index = IsLittleEndian ? 0 : 8 - Size; emitBytes(StringRef(reinterpret_cast(&Swapped) + Index, Size)); } -void MCStreamer::emitIntValue(APInt Value) { +void MCStreamer::emitIntValue(const APInt &Value) { if (Value.getNumWords() == 1) { emitIntValue(Value.getLimitedValue(), Value.getBitWidth() / 8); return; -- GitLab From 2f1f6b704d83f87be7ea885480caf1c86d8cfaee Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:58:25 +0800 Subject: [PATCH 011/879] [LLVM] Use `std::move` for APInt. NFC. (#86257) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=ba3e326def3a6e5cd6d72ff5a49c74fba18de1df&stat=instructions:u --- llvm/include/llvm/Analysis/InlineCost.h | 3 ++- llvm/include/llvm/Analysis/MemoryBuiltins.h | 6 ++++-- llvm/lib/Analysis/ConstantFolding.cpp | 2 +- llvm/lib/Analysis/InstructionSimplify.cpp | 6 ++++-- llvm/lib/IR/LLVMContextImpl.h | 2 +- llvm/lib/Transforms/Scalar/MergeICmps.cpp | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 3a760e0a85ce..c5978ce54fc1 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -65,7 +65,8 @@ const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize"; // The cost-benefit pair computed by cost-benefit analysis. class CostBenefitPair { public: - CostBenefitPair(APInt Cost, APInt Benefit) : Cost(Cost), Benefit(Benefit) {} + CostBenefitPair(APInt Cost, APInt Benefit) + : Cost(std::move(Cost)), Benefit(std::move(Benefit)) {} const APInt &getCost() const { return Cost; } diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index da092866de65..bb282a1b73d3 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -196,7 +196,8 @@ public: T Offset; SizeOffsetType() = default; - SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {} + SizeOffsetType(T Size, T Offset) + : Size(std::move(Size)), Offset(std::move(Offset)) {} bool knownSize() const { return C::known(Size); } bool knownOffset() const { return C::known(Offset); } @@ -215,7 +216,8 @@ public: /// \p APInts. struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; - SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} + SizeOffsetAPInt(APInt Size, APInt Offset) + : SizeOffsetType(std::move(Size), std::move(Offset)) {} static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index 6139b5be85be..749374a3aa48 100644 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -751,7 +751,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL) { APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0); - return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL); + return ConstantFoldLoadFromConstPtr(C, Ty, std::move(Offset), DL); } Constant *llvm::ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty, diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 7a37ae86c7f3..9ff3faff7990 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6115,7 +6115,8 @@ static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, if (OffsetInt.srem(4) != 0) return nullptr; - Constant *Loaded = ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, OffsetInt, DL); + Constant *Loaded = + ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, std::move(OffsetInt), DL); if (!Loaded) return nullptr; @@ -6983,7 +6984,8 @@ Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp, if (PtrOp == GV) { // Index size may have changed due to address space casts. Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType())); - return ConstantFoldLoadFromConstPtr(GV, LI->getType(), Offset, Q.DL); + return ConstantFoldLoadFromConstPtr(GV, LI->getType(), std::move(Offset), + Q.DL); } return nullptr; diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index 58e0f21244f7..7c67e191348e 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -441,7 +441,7 @@ template <> struct MDNodeKeyImpl { bool IsUnsigned; MDNodeKeyImpl(APInt Value, bool IsUnsigned, MDString *Name) - : Value(Value), Name(Name), IsUnsigned(IsUnsigned) {} + : Value(std::move(Value)), Name(Name), IsUnsigned(IsUnsigned) {} MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name) : Value(APInt(64, Value, !IsUnsigned)), Name(Name), IsUnsigned(IsUnsigned) {} diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp index 1e0906717549..2bd13556c696 100644 --- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp +++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp @@ -74,7 +74,7 @@ namespace { struct BCEAtom { BCEAtom() = default; BCEAtom(GetElementPtrInst *GEP, LoadInst *LoadI, int BaseId, APInt Offset) - : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(Offset) {} + : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(std::move(Offset)) {} BCEAtom(const BCEAtom &) = delete; BCEAtom &operator=(const BCEAtom &) = delete; -- GitLab From ef57977f2aa32661a09fa6538f47ddee0a004d11 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 15:02:27 +0800 Subject: [PATCH 012/879] [NewPM][Hexagon] Add HexagonPassRegistry.def (#86244) Prepare for dag-isel, also migrate some test case --- .../Target/Hexagon/HexagonPassRegistry.def | 21 +++++++++++++++++++ .../Target/Hexagon/HexagonTargetMachine.cpp | 3 +++ .../hexagon_vector_loop_carried_reuse.ll | 2 +- ...n_vector_loop_carried_reuse_commutative.ll | 2 +- ...agon_vector_loop_carried_reuse_constant.ll | 2 +- ...xagon_vector_loop_carried_reuse_invalid.ll | 2 +- .../CodeGen/Hexagon/hvx-loopidiom-memcpy.ll | 1 + .../Hexagon/loop-idiom/hexagon-memmove1.ll | 2 ++ .../Hexagon/loop-idiom/hexagon-memmove2.ll | 2 ++ llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll | 2 +- .../Hexagon/loop-idiom/memmove-rt-check.ll | 1 + .../Hexagon/loop-idiom/nullptr-crash.ll | 1 + .../Hexagon/loop-idiom/pmpy-infinite-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-long-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-shiftconv-fail.ll | 1 + llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll | 2 ++ 16 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 llvm/lib/Target/Hexagon/HexagonPassRegistry.def diff --git a/llvm/lib/Target/Hexagon/HexagonPassRegistry.def b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def new file mode 100644 index 000000000000..4f58ae6193e0 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def @@ -0,0 +1,21 @@ +//===- HexagonPassRegistry.def - Registry of Hexagon passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// Hexagon backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef LOOP_PASS +#define LOOP_PASS(NAME, CREATE_PASS) +#endif +LOOP_PASS("hexagon-loop-idiom", HexagonLoopIdiomRecognitionPass()) +LOOP_PASS("hexagon-vlcr", HexagonVectorLoopCarriedReusePass()) +#undef LOOP_PASS diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 3c346c334d6d..e64d7e52a9aa 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -299,6 +299,9 @@ HexagonTargetMachine::getSubtargetImpl(const Function &F) const { void HexagonTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { +#define GET_PASS_REGISTRY "HexagonPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerLateLoopOptimizationsEPCallback( [=](LoopPassManager &LPM, OptimizationLevel Level) { LPM.addPass(HexagonLoopIdiomRecognitionPass()); diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll index 0771fda02cfb..7ccee1689185 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll index 25afb9f1a137..532f7fd06793 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -march=hexagon -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %v32.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll index 53973423732c..ecfcf53d9133 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK-NOT: %.hexagon.vlcr ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll index b440dba66f67..9872faeb3da0 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll @@ -1,4 +1,4 @@ -; RUN: opt -hexagon-vlcr < %s -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes=hexagon-vlcr -S %s | FileCheck %s ; Test that reuse doesn't occur due to two shufflevectors with different masks. diff --git a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll index ab7bf1b4b0e8..c53e57800edd 100644 --- a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll +++ b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -mtriple=hexagon-- -p hexagon-loop-idiom -disable-memcpy-idiom -S < %s | FileCheck %s ; Make sure we don't convert load/store loops into memcpy if the access type ; is a vector. Using vector instructions is generally better in such cases. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll index c7110263c658..5ace9e6ee486 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll @@ -1,6 +1,8 @@ ; Check for recognizing the "memmove" idiom. ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s ; CHECK: call void @llvm.memmove ; Function Attrs: norecurse nounwind diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll index 234e4f56b5d8..ed56a332f657 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s define void @PR14241(ptr %s, i64 %size) #0 { ; Ensure that we don't form a memcpy for strided loops. Briefly, when we taught diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll index 140c676175ea..e5bcc2b9aebf 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -hexagon-loop-idiom < %s | opt -S -passes='loop(loop-deletion),gvn' +; RUN: opt -mtriple hexagon-- -S -passes='loop(hexagon-loop-idiom,loop-deletion),gvn' ; REQUIRES: asserts ; This tests that the HexagonLoopIdiom pass does not mark LCSSA information diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll index 7a7d1d9b1a86..78f0c9e36b55 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S < %s | FileCheck %s ; Make sure that we generate correct runtime checks. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll index 37e1bb6eb7df..ce02b62911c0 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s +; RUN: opt -p hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s ; REQUIRES: asserts target triple = "hexagon" diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll index 1934ced7e7ae..74c02d63d54d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; CHECK-LABEL: define void @fred ; Check that this test does not crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll index b25010f2a90f..94b0c96c3d51 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; ; The number of nested selects caused the simplification loop to take ; more than the maximum number of iterations. This caused the compiler diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll index e4b2b5a298ed..a00b1d5876ba 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; REQUIRES: asserts ; ; Check for sane output, this used to crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll index 781618e58901..2461e1cfde8d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ +; RUN: | FileCheck %s target triple = "hexagon" -- GitLab From 691b97c884a15a7eac641ddf67c9f2f30fb4e747 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 01:02:08 -0700 Subject: [PATCH 013/879] [ELF] Remove zero-value DT_JMPREL when IPLT is present while PLT isn't The zero-value DT_JMPREL is benign but not needed. This is also code simplification available after https://reviews.llvm.org/D65651 --- lld/ELF/SyntheticSections.cpp | 16 ++------------- .../ELF/aarch64-gnu-ifunc-nonpreemptable.s | 12 +++++------ lld/test/ELF/gnu-ifunc-dyntags.s | 8 +++----- lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s | 8 ++++---- lld/test/ELF/riscv-ifunc-nonpreemptible.s | 20 +++++++++---------- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 248ff6b4a865..10eda17f0cb3 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1285,13 +1285,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { // output section. When this occurs we cannot just use the OutputSection // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to // overlap with the [DT_RELA, DT_RELA + DT_RELASZ). -static uint64_t addPltRelSz() { - size_t size = in.relaPlt->getSize(); - if (in.relaIplt->getParent() == in.relaPlt->getParent() && - in.relaIplt->name == in.relaPlt->name) - size += in.relaIplt->getSize(); - return size; -} +static uint64_t addPltRelSz() { return in.relaPlt->getSize(); } // Add remaining entries to complete .dynamic contents. template @@ -1407,13 +1401,7 @@ DynamicSection::computeContents() { addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT, sizeof(Elf_Relr)); } - // .rel[a].plt section usually consists of two parts, containing plt and - // iplt relocations. It is possible to have only iplt relocations in the - // output. In that case relaPlt is empty and have zero offset, the same offset - // as relaIplt has. And we still want to emit proper dynamic tags for that - // case, so here we always use relaPlt as marker for the beginning of - // .rel[a].plt section. - if (isMain && (in.relaPlt->isNeeded() || in.relaIplt->isNeeded())) { + if (isMain && in.relaPlt->isNeeded()) { addInSec(DT_JMPREL, *in.relaPlt); entries.emplace_back(DT_PLTRELSZ, addPltRelSz()); switch (config->emachine) { diff --git a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s index 4f33dde9d1a9..54a27a8cc0d9 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s +++ b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s @@ -65,14 +65,14 @@ main: # PIE-EMPTY: # PIE-NEXT: : # PIE-NEXT: 10270: adrp x16, 0x30000 -# PIE-NEXT: 10274: ldr x17, [x16, #896] -# PIE-NEXT: 10278: add x16, x16, #896 -# PIE-NEXT: 1027c: br x17 +# PIE-NEXT: ldr x17, [x16, #832] +# PIE-NEXT: add x16, x16, #832 +# PIE-NEXT: br x17 # PIE-RELOC: .rela.dyn { -# PIE-RELOC-NEXT: 0x30380 R_AARCH64_IRELATIVE - 0x10260 +# PIE-RELOC-NEXT: 0x30340 R_AARCH64_IRELATIVE - 0x10260 # PIE-RELOC-NEXT: } # PIE-RELOC: Hex dump of section '.got.plt': -# NO-APPLY: 0x00030380 00000000 00000000 -# APPLY: 0x00030380 60020100 00000000 +# NO-APPLY: 0x00030340 00000000 00000000 +# APPLY: 0x00030340 60020100 00000000 # PIE-RELOC-EMPTY: diff --git a/lld/test/ELF/gnu-ifunc-dyntags.s b/lld/test/ELF/gnu-ifunc-dyntags.s index fd80dc24f2f8..57a17245a3e8 100644 --- a/lld/test/ELF/gnu-ifunc-dyntags.s +++ b/lld/test/ELF/gnu-ifunc-dyntags.s @@ -9,15 +9,13 @@ # CHECK: Name Size VMA # CHECK: .rela.dyn 00000030 0000000000000248 -# CHECK: .got.plt 00000010 00000000000033b0 +# CHECK: .got.plt 00000010 0000000000003370 # TAGS: Tag Type Name/Value # TAGS: 0x0000000000000007 RELA 0x248 # TAGS: 0x0000000000000008 RELASZ 48 (bytes) -# TAGS: 0x0000000000000017 JMPREL 0x0 -# TAGS: 0x0000000000000002 PLTRELSZ 0 (bytes) -# TAGS: 0x0000000000000003 PLTGOT 0x33B0 -# TAGS: 0x0000000000000014 PLTREL RELA +# TAGS-NOT: JMPREL +# TAGS-NOT: PLTREL # TAGS: Relocations [ # TAGS-NEXT: Section {{.*}} .rela.dyn { diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s index a93f3cecb0c6..c9a0381b610a 100644 --- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s +++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s @@ -10,16 +10,16 @@ # RUN: llvm-readelf -x .got2 %t | FileCheck --check-prefix=HEX2 %s # RELOC: .rela.dyn { -# RELOC-NEXT: 0x3024C R_PPC_RELATIVE - 0x101A0 -# RELOC-NEXT: 0x30250 R_PPC_IRELATIVE - 0x10188 +# RELOC-NEXT: 0x3022C R_PPC_RELATIVE - 0x101A0 +# RELOC-NEXT: 0x30230 R_PPC_IRELATIVE - 0x10188 # RELOC-NEXT: } # SYM: 000101a0 0 FUNC GLOBAL DEFAULT {{.*}} func # HEX: Hex dump of section '.got2': -# HEX-NEXT: 0x0003024c 00000000 .... +# HEX-NEXT: 0x0003022c 00000000 .... # HEX2: Hex dump of section '.got2': -# HEX2-NEXT: 0x0003024c 000101a0 .... +# HEX2-NEXT: 0x0003022c 000101a0 .... .section .got2,"aw" .long func diff --git a/lld/test/ELF/riscv-ifunc-nonpreemptible.s b/lld/test/ELF/riscv-ifunc-nonpreemptible.s index 21c607545110..eda5548eef8b 100644 --- a/lld/test/ELF/riscv-ifunc-nonpreemptible.s +++ b/lld/test/ELF/riscv-ifunc-nonpreemptible.s @@ -16,11 +16,11 @@ # RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefix=DIS64 %s # RELOC32: .rela.dyn { -# RELOC32-NEXT: 0x3220 R_RISCV_IRELATIVE - 0x117C +# RELOC32-NEXT: 0x3200 R_RISCV_IRELATIVE - 0x117C # RELOC32-NEXT: } # RELOC32-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC32: 0x00003220 00000000 -# APPLY-RELOC32: 0x00003220 7c110000 +# NO-APPLY-RELOC32: 0x00003200 00000000 +# APPLY-RELOC32: 0x00003200 7c110000 # RELOC32-EMPTY: # SYM32: 0001190 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -30,18 +30,18 @@ # DIS32-NEXT: addi a0, a0, 0x10 # DIS32: Disassembly of section .iplt: # DIS32: : -## 32-bit: &.got.plt[func]-. = 0x3220-0x1190 = 4096*2+144 +## 32-bit: &.got.plt[func]-. = 0x3200-0x1190 = 4096*2+0x70 # DIS32-NEXT: 1190: auipc t3, 0x2 -# DIS32-NEXT: lw t3, 0x90(t3) +# DIS32-NEXT: lw t3, 0x70(t3) # DIS32-NEXT: jalr t1, t3 # DIS32-NEXT: nop # RELOC64: .rela.dyn { -# RELOC64-NEXT: 0x3380 R_RISCV_IRELATIVE - 0x1260 +# RELOC64-NEXT: 0x3340 R_RISCV_IRELATIVE - 0x1260 # RELOC64-NEXT: } # RELOC64-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC64: 0x00003380 00000000 00000000 -# APPLY-RELOC64: 0x00003380 60120000 00000000 +# NO-APPLY-RELOC64: 0x00003340 00000000 00000000 +# APPLY-RELOC64: 0x00003340 60120000 00000000 # RELOC64-EMPTY: # SYM64: 000000000001270 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -51,9 +51,9 @@ # DIS64-NEXT: addi a0, a0, 0xc # DIS64: Disassembly of section .iplt: # DIS64: : -## 64-bit: &.got.plt[func]-. = 0x3380-0x1270 = 4096*2+272 +## 64-bit: &.got.plt[func]-. = 0x3340-0x1270 = 4096*2+0xd0 # DIS64-NEXT: 1270: auipc t3, 0x2 -# DIS64-NEXT: ld t3, 0x110(t3) +# DIS64-NEXT: ld t3, 0xd0(t3) # DIS64-NEXT: jalr t1, t3 # DIS64-NEXT: nop -- GitLab From 579dc7f8441a8044b92bdfa6f0db2f91301c0eed Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:03:45 -0700 Subject: [PATCH 014/879] [clang-forma] Support `PointerAlignment` for pointers to members (#86253) Fixes #85761. --- clang/lib/Format/TokenAnnotator.cpp | 37 +++++++++++------- clang/unittests/Format/FormatTest.cpp | 38 ++++++++++++------- clang/unittests/Format/QualifierFixerTest.cpp | 36 +++++++++--------- 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index a5cafcbacaa5..7757c8ff7639 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -4357,9 +4357,11 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Left.is(tok::kw_auto) && Right.isOneOf(tok::l_paren, tok::l_brace)) return false; + const auto *BeforeLeft = Left.Previous; + // operator co_await(x) - if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && Left.Previous && - Left.Previous->is(tok::kw_operator)) { + if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && BeforeLeft && + BeforeLeft->is(tok::kw_operator)) { return false; } // co_await (x), co_yield (x), co_return (x) @@ -4394,8 +4396,10 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, } if (Left.is(tok::colon)) return Left.isNot(TT_ObjCMethodExpr); - if (Left.is(tok::coloncolon)) - return false; + if (Left.is(tok::coloncolon)) { + return Right.is(tok::star) && Right.is(TT_PointerOrReference) && + Style.PointerAlignment != FormatStyle::PAS_Left; + } if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less)) { if (Style.Language == FormatStyle::LK_TextProto || (Style.Language == FormatStyle::LK_Proto && @@ -4410,8 +4414,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return false; } if (Right.is(tok::ellipsis)) { - return Left.Tok.isLiteral() || (Left.is(tok::identifier) && Left.Previous && - Left.Previous->is(tok::kw_case)); + return Left.Tok.isLiteral() || (Left.is(tok::identifier) && BeforeLeft && + BeforeLeft->is(tok::kw_case)); } if (Left.is(tok::l_square) && Right.is(tok::amp)) return Style.SpacesInSquareBrackets; @@ -4479,8 +4483,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Right.is(tok::l_brace) && Right.is(BK_Block)) return true; // for (auto a = 0, b = 0; const auto& c : {1, 2, 3}) - if (Left.Previous && Left.Previous->isTypeOrIdentifier(IsCpp) && - Right.Next && Right.Next->is(TT_RangeBasedForLoopColon)) { + if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next && + Right.Next->is(TT_RangeBasedForLoopColon)) { return getTokenPointerOrReferenceAlignment(Left) != FormatStyle::PAS_Right; } @@ -4502,12 +4506,17 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, startsWithInitStatement(Line)))) { return false; } - return Left.Previous && !Left.Previous->isOneOf( - tok::l_paren, tok::coloncolon, tok::l_square); + if (!BeforeLeft) + return false; + if (BeforeLeft->is(tok::coloncolon)) { + return Left.is(tok::star) && + Style.PointerAlignment != FormatStyle::PAS_Right; + } + return !BeforeLeft->isOneOf(tok::l_paren, tok::l_square); } // Ensure right pointer alignment with ellipsis e.g. int *...P - if (Left.is(tok::ellipsis) && Left.Previous && - Left.Previous->isPointerOrReference()) { + if (Left.is(tok::ellipsis) && BeforeLeft && + BeforeLeft->isPointerOrReference()) { return Style.PointerAlignment != FormatStyle::PAS_Right; } @@ -4669,13 +4678,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return Style.SpaceBeforeParensOptions.AfterFunctionDefinitionName || spaceRequiredBeforeParens(Right); } - if (!Left.Previous || !Left.Previous->isOneOf(tok::period, tok::arrow)) { + if (!BeforeLeft || !BeforeLeft->isOneOf(tok::period, tok::arrow)) { if (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch)) { return Style.SpaceBeforeParensOptions.AfterControlStatements || spaceRequiredBeforeParens(Right); } if (Left.isOneOf(tok::kw_new, tok::kw_delete)) { - return ((!Line.MightBeFunctionDecl || !Left.Previous) && + return ((!Line.MightBeFunctionDecl || !BeforeLeft) && Style.SpaceBeforeParens != FormatStyle::SBPO_Never) || spaceRequiredBeforeParens(Right); } diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index bea989c8c306..cf8d6ab691d9 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -3621,8 +3621,8 @@ TEST_F(FormatTest, FormatsClasses) { " : public aaaaaaaaaaaaaaaaaaa {};"); verifyFormat("template \n" - "struct Aaaaaaaaaaaaaaaaa\n" - " : Aaaaaaaaaaaaaaaaa {};"); + "struct Aaaaaaaaaaaaaaaaa\n" + " : Aaaaaaaaaaaaaaaaa {};"); verifyFormat("class ::A::B {};"); } @@ -11034,10 +11034,10 @@ TEST_F(FormatTest, UnderstandsBinaryOperators) { } TEST_F(FormatTest, UnderstandsPointersToMembers) { - verifyFormat("int A::*x;"); - verifyFormat("int (S::*func)(void *);"); - verifyFormat("void f() { int (S::*func)(void *); }"); - verifyFormat("typedef bool *(Class::*Member)() const;"); + verifyFormat("int A:: *x;"); + verifyFormat("int (S:: *func)(void *);"); + verifyFormat("void f() { int (S:: *func)(void *); }"); + verifyFormat("typedef bool *(Class:: *Member)() const;"); verifyFormat("void f() {\n" " (a->*f)();\n" " a->*x;\n" @@ -11052,9 +11052,19 @@ TEST_F(FormatTest, UnderstandsPointersToMembers) { verifyFormat( "(aaaaaaaaaa->*bbbbbbb)(\n" " aaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa));"); + FormatStyle Style = getLLVMStyle(); + EXPECT_EQ(Style.PointerAlignment, FormatStyle::PAS_Right); + verifyFormat("typedef bool *(Class:: *Member)() const;", Style); + verifyFormat("void f(int A:: *p) { int A:: *v = &A::B; }", Style); + Style.PointerAlignment = FormatStyle::PAS_Left; - verifyFormat("typedef bool* (Class::*Member)() const;", Style); + verifyFormat("typedef bool* (Class::* Member)() const;", Style); + verifyFormat("void f(int A::* p) { int A::* v = &A::B; }", Style); + + Style.PointerAlignment = FormatStyle::PAS_Middle; + verifyFormat("typedef bool * (Class:: * Member)() const;", Style); + verifyFormat("void f(int A:: * p) { int A:: * v = &A::B; }", Style); } TEST_F(FormatTest, UnderstandsUnaryOperators) { @@ -12386,7 +12396,7 @@ TEST_F(FormatTest, FormatsFunctionTypes) { verifyFormat("int (*func)(void *);"); verifyFormat("void f() { int (*func)(void *); }"); verifyFormat("template \n" - "using MyCallback = void (CallbackClass::*)(SomeObject *Data);"); + "using Callback = void (CallbackClass:: *)(SomeObject *Data);"); verifyGoogleFormat("A;"); verifyGoogleFormat("void* (*a)(int);"); @@ -19149,13 +19159,13 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { "int bbbbbbb = 0;", Alignment); // http://llvm.org/PR68079 - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int (A::*)() &;\n" - "using RRFn = int (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int (A:: *)() &;\n" + "using RRFn = int (A:: *)() &&;", Alignment); - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int *(A::*)() &;\n" - "using RRFn = double (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int *(A:: *)() &;\n" + "using RRFn = double (A:: *)() &&;", Alignment); // PAS_Right diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp index 43476aea6633..792d8f3c3a98 100644 --- a/clang/unittests/Format/QualifierFixerTest.cpp +++ b/clang/unittests/Format/QualifierFixerTest.cpp @@ -305,7 +305,7 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("Foo inline static const;", "Foo inline const static;", Style); verifyFormat("Foo inline static const;", Style); - verifyFormat("Foo::Bar const volatile A::*;", + verifyFormat("Foo::Bar const volatile A:: *;", "volatile const Foo::Bar A::*;", Style); @@ -523,14 +523,15 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("const INTPTR a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("int const S::*a;", "const int S:: *a;", Style); - verifyFormat("int const S::*const a;", "const int S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("float const (C::*p)(int);", "const float (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("int const S:: *a;", "const int S:: *a;", Style); + verifyFormat("int const S:: *const a;", "const int S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("float const (C:: *p)(int);", "const float (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, LeftQualifier) { @@ -830,14 +831,15 @@ TEST_F(QualifierFixerTest, LeftQualifier) { verifyFormat("INTPTR const a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("const int S::*a;", "int const S:: *a;", Style); - verifyFormat("const int S::*const a;", "int const S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("const float (C::*p)(int);", "float const (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("const int S:: *a;", "int const S:: *a;", Style); + verifyFormat("const int S:: *const a;", "int const S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("const float (C:: *p)(int);", "float const (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, ConstVolatileQualifiersOrder) { -- GitLab From 47423e9827abfdcc6b10ce41618965861b0e69a4 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:24:56 -0700 Subject: [PATCH 015/879] [clang-format][NFC] Clean up IsQualifiedPointerOrReference in TokenAnnotator --- clang/lib/Format/TokenAnnotator.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 7757c8ff7639..4c83a7a3a323 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -2753,10 +2753,9 @@ private: } // Heuristically try to determine whether the parentheses contain a type. - auto IsQualifiedPointerOrReference = [this](FormatToken *T) { + auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) { // This is used to handle cases such as x = (foo *const)&y; assert(!T->isTypeName(IsCpp) && "Should have already been checked"); - (void)IsCpp; // Avoid -Wunused-lambda-capture when assertion is disabled. // Strip trailing qualifiers such as const or volatile when checking // whether the parens could be a cast to a pointer/reference type. while (T) { @@ -2789,7 +2788,7 @@ private: !Tok.Previous || Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) || Tok.Previous->isTypeName(IsCpp) || - IsQualifiedPointerOrReference(Tok.Previous); + IsQualifiedPointerOrReference(Tok.Previous, IsCpp); bool ParensCouldEndDecl = Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater); if (ParensAreType && !ParensCouldEndDecl) -- GitLab From f317fd266c184bc8f9f4d7e8e03c043d3406333a Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 15:30:26 +0500 Subject: [PATCH 016/879] Revert "[mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225)" This reverts commit 01b1b0c1f728e2c2639edc654424f50830295989. Breaks following AArch64 SVE buildbots: https://lab.llvm.org/buildbot/#/builders/184/builds/11363 https://lab.llvm.org/buildbot/#/builders/176/builds/9331 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir deleted file mode 100644 index 57d69383c2de..000000000000 --- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir +++ /dev/null @@ -1,60 +0,0 @@ -// DEFINE: %{compile} = mlir-opt %s \ -// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ -// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ -// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t -// DEFINE: %{entry_point} = conv -// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ -// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils - -// RUN: %{compile} | %{run} | FileCheck %s - -func.func @conv() { - // Define input/output tensors - %input_init = tensor.empty() : tensor<1x8x6xi32> - %output_init = tensor.empty() : tensor<1x7x6xi32> - - %five = arith.constant 5 : i32 - %zero = arith.constant 0 : i32 - %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> - %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> - - // Define the filter tensor - %filter = arith.constant dense<[ - [ 1, 2, 3, 4, 5, 6], - [ 11, 12, 13, 14, 15, 16] - ]> : tensor<2x6xi32> - - // static sizes -> dynamic sizes - %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> - %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> - %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> - - // Run the convolution - %res = linalg.depthwise_conv_1d_nwc_wc - ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) - outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> - - // Print the results - // CHECK: SVE: START OF TEST OUTPUT - vector.print str "SVE: START OF TEST OUTPUT\n" - - // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = - // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] - %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> - call @printMemrefI32(%xf) : (tensor<*xi32>) -> () - - // CHECK-NEXT: SVE: END OF TEST OUTPUT - vector.print str "SVE: END OF TEST OUTPUT\n" - - return -} - -module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { - %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op - transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op - transform.yield - } -} - -func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From d7c672834ec863b458af8ca493157e1e31aaf480 Mon Sep 17 00:00:00 2001 From: XChy Date: Sat, 23 Mar 2024 19:00:42 +0800 Subject: [PATCH 017/879] [CodeGen][NFC] Update tests in AArch64/and-sink.ll --- llvm/test/CodeGen/AArch64/and-sink.ll | 60 +++++++++++++++++++++------ 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll index 4d085869de24..f298a55dab72 100644 --- a/llvm/test/CodeGen/AArch64/and-sink.ll +++ b/llvm/test/CodeGen/AArch64/and-sink.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s ; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s ; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s @@ -9,9 +10,18 @@ ; Test that and is sunk into cmp block to form tbz. define dso_local i32 @and_sink1(i32 %a, i1 %c) { ; CHECK-LABEL: and_sink1: -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: tbz w1, #0, .LBB0_3 +; CHECK-NEXT: // %bb.1: // %bb0 +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: tbnz w0, #2, .LBB0_3 +; CHECK-NEXT: // %bb.2: +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB0_3: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink1( ; CHECK-CGP-NOT: and i32 @@ -35,12 +45,30 @@ bb2: ; Test that both 'and' and cmp get sunk to form tbz. define dso_local i32 @and_sink2(i32 %a, i1 %c, i1 %c2) { ; CHECK-LABEL: and_sink2: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:B] -; CHECK: tbz w2, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:C] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, wzr +; CHECK-NEXT: adrp x9, A +; CHECK-NEXT: str wzr, [x9, :lo12:A] +; CHECK-NEXT: tbz w1, #0, .LBB1_5 +; CHECK-NEXT: // %bb.1: // %bb0.preheader +; CHECK-NEXT: adrp x8, B +; CHECK-NEXT: adrp x9, C +; CHECK-NEXT: .LBB1_2: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:B] +; CHECK-NEXT: tbz w2, #0, .LBB1_6 +; CHECK-NEXT: // %bb.3: // %bb1 +; CHECK-NEXT: // in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: str wzr, [x9, :lo12:C] +; CHECK-NEXT: tbnz w0, #2, .LBB1_2 +; CHECK-NEXT: // %bb.4: +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: .LBB1_5: // %common.ret +; CHECK-NEXT: mov w0, w8 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB1_6: +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink2( ; CHECK-CGP-NOT: and i32 @@ -71,10 +99,16 @@ bb3: ; Test that 'and' is not sunk since cbz is a better alternative. define dso_local i32 @and_sink3(i32 %a) { ; CHECK-LABEL: and_sink3: -; CHECK: and [[REG:w[0-9]+]], w0, #0x3 -; CHECK: [[LOOP:.L[A-Z0-9_]+]]: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: cbz [[REG]], [[LOOP]] +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: and w9, w0, #0x3 +; CHECK-NEXT: .LBB2_1: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: cbz w9, .LBB2_1 +; CHECK-NEXT: // %bb.2: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink3( ; CHECK-CGP-NEXT: and i32 -- GitLab From d365a45cb3eaa640b09874fb7984a6a69683c773 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Sat, 23 Mar 2024 15:12:44 +0300 Subject: [PATCH 018/879] [GlobalISel] Introduce G_TRAP, G_DEBUGTRAP, G_UBSANTRAP (#84941) Here we introduce three new GMIR instructions to cover a set of trap intrinsics. The idea behind it is that generic intrinsics shouldn't be used with G_INTRINSIC opcode. These new instructions can match perfectly with existing trap ISD nodes. It allows X86, AArch64, RISCV and Mips to reuse SelectionDAG patterns for selection and avoid manual selection. However AMDGPU is an exception. It selects traps during legalization regardless SelectionDAG or GlobalISel. Since there are not many places where traps are used, this change attempts to clean up all the usages of G_INTRINSIC with trap intrinsics. So, there is no stage when both G_TRAP and G_INTRINSIC_W_SIDE_EFFECTS(@llvm.trap) are allowed. --- llvm/docs/GlobalISel/GenericOpcode.rst | 19 +++++ llvm/docs/LangRef.rst | 6 ++ .../llvm/CodeGen/GlobalISel/IRTranslator.h | 4 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 5 ++ llvm/include/llvm/Support/TargetOpcodes.def | 5 ++ llvm/include/llvm/Target/GenericOpcodes.td | 22 ++++++ .../Target/GlobalISel/SelectionDAGCompat.td | 3 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 47 ++++++++---- llvm/lib/CodeGen/MachineVerifier.cpp | 11 +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 + .../GISel/AArch64InstructionSelector.cpp | 20 ++--- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 23 +++--- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 8 +- llvm/lib/Target/Mips/MipsLegalizerInfo.cpp | 8 -- .../RISCV/GISel/RISCVInstructionSelector.cpp | 27 ------- .../X86/GISel/X86InstructionSelector.cpp | 19 ----- .../GlobalISel/irtranslator-unreachable.ll | 2 +- .../AArch64/GlobalISel/legalize-exceptions.ll | 2 +- .../GlobalISel/legalizer-info-validation.mir | 9 +++ .../AArch64/GlobalISel/select-trap.mir | 2 +- .../AArch64/GlobalISel/uaddo-8-16-bits.mir | 52 ++++++------- .../AMDGPU/GlobalISel/legalize-trap.mir | 4 +- .../trap.mir | 9 ++- .../GlobalISel/instruction-select/trap.mir | 4 +- .../X86/GlobalISel/x86-select-trap.mir | 2 +- llvm/test/CodeGen/X86/isel-traps.ll | 73 +++++++++++++++++++ .../test/MachineVerifier/test_g_ubsantrap.mir | 18 +++++ 27 files changed, 276 insertions(+), 131 deletions(-) rename llvm/test/CodeGen/Mips/GlobalISel/{legalizer => instruction-select}/trap.mir (55%) create mode 100644 llvm/test/CodeGen/X86/isel-traps.ll create mode 100644 llvm/test/MachineVerifier/test_g_ubsantrap.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index ac6217d08e6a..cae2c21b80d7 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -939,6 +939,25 @@ The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`. Unlike SelectionDAG, there is no _VOID variant. Both of these are permitted to have zero, one, or multiple results. +G_TRAP, G_DEBUGTRAP, G_UBSANTRAP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Represents :ref:`llvm.trap `, :ref:`llvm.debugtrap ` +and :ref:`llvm.ubsantrap ` that generate a target dependent +trap instructions. + +.. code-block:: none + + G_TRAP + +.. code-block:: none + + G_DEBUGTRAP + +.. code-block:: none + + G_UBSANTRAP 12 + Variadic Arguments ------------------ diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 8bc1cab01bf0..391fd2d960dd 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -26926,6 +26926,8 @@ Arguments: The argument should be an MDTuple containing any number of MDStrings. +.. _llvm.trap: + '``llvm.trap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26953,6 +26955,8 @@ This intrinsic is lowered to the target dependent trap instruction. If the target does not have a trap instruction, this intrinsic will be lowered to a call of the ``abort()`` function. +.. _llvm.debugtrap: + '``llvm.debugtrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26980,6 +26984,8 @@ This intrinsic is lowered to code which is intended to cause an execution trap with the intention of requesting the attention of a debugger. +.. _llvm.ubsantrap: + '``llvm.ubsantrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 6ae7c1440907..5f28908e998a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -243,6 +243,10 @@ private: bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode); + /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap). + bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder, + unsigned Opcode); + // Translate @llvm.experimental.vector.interleave2 and // @llvm.experimental.vector.deinterleave2 intrinsics for fixed-width vector // types into vector shuffles. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index aaa81342845b..3ba036ae713f 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -2113,6 +2113,11 @@ public: DstMMO, SrcMMO); } + /// Build and insert G_TRAP or G_DEBUGTRAP + MachineInstrBuilder buildTrap(bool Debug = false) { + return buildInstr(Debug ? TargetOpcode::G_DEBUGTRAP : TargetOpcode::G_TRAP); + } + /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width. MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src, const SrcOp &LSB, const SrcOp &Width) { diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 899eaad5842a..5765926d6d93 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -837,6 +837,11 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE) HANDLE_TARGET_OPCODE(G_MEMSET) HANDLE_TARGET_OPCODE(G_BZERO) +/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics +HANDLE_TARGET_OPCODE(G_TRAP) +HANDLE_TARGET_OPCODE(G_DEBUGTRAP) +HANDLE_TARGET_OPCODE(G_UBSANTRAP) + /// Vector reductions HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD) HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 67d405ba96fa..d0f471eb29b6 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1575,6 +1575,28 @@ def G_BZERO : GenericInstruction { let mayStore = true; } +//------------------------------------------------------------------------------ +// Trap intrinsics +//------------------------------------------------------------------------------ +def G_TRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; + let mayStore = true; +} + +def G_DEBUGTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; +} + +def G_UBSANTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i8imm:$kind); + let hasSideEffects = true; +} + //------------------------------------------------------------------------------ // Bitfield extraction. //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index b1f3c500a1b6..4bb9929e2cf8 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -250,6 +250,9 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 757af3b1c4fe..ef95ceec18a1 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1771,6 +1771,32 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, return true; } +bool IRTranslator::translateTrap(const CallInst &CI, + MachineIRBuilder &MIRBuilder, + unsigned Opcode) { + StringRef TrapFuncName = + CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); + if (TrapFuncName.empty()) { + if (Opcode == TargetOpcode::G_UBSANTRAP) { + uint64_t Code = cast(CI.getOperand(0))->getZExtValue(); + MIRBuilder.buildInstr(Opcode, {}, ArrayRef{Code}); + } else { + MIRBuilder.buildInstr(Opcode); + } + return true; + } + + CallLowering::CallLoweringInfo Info; + if (Opcode == TargetOpcode::G_UBSANTRAP) + Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), + CI.getArgOperand(0)->getType(), 0}); + + Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); + Info.CB = &CI; + Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; + return CLI->lowerCall(MIRBuilder, Info); +} + bool IRTranslator::translateVectorInterleave2Intrinsic( const CallInst &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 && @@ -2459,22 +2485,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return true; } case Intrinsic::trap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP); case Intrinsic::debugtrap: - case Intrinsic::ubsantrap: { - StringRef TrapFuncName = - CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); - if (TrapFuncName.empty()) - break; // Use the default handling. - CallLowering::CallLoweringInfo Info; - if (ID == Intrinsic::ubsantrap) { - Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), - CI.getArgOperand(0)->getType(), 0}); - } - Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); - Info.CB = &CI; - Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; - return CLI->lowerCall(MIRBuilder, Info); - } + return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP); + case Intrinsic::ubsantrap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP); case Intrinsic::amdgcn_cs_chain: return translateCallBase(CI, MIRBuilder); case Intrinsic::fptrunc_round: { @@ -3047,7 +3062,7 @@ bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuil } } - MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef()); + MIRBuilder.buildTrap(); return true; } diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 005efe48ac0c..9a1498d4070e 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1867,6 +1867,17 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } + case TargetOpcode::G_UBSANTRAP: { + const MachineOperand &KindOp = MI->getOperand(0); + if (!MI->getOperand(0).isImm()) { + report("Crash kind must be an immediate", &KindOp, 0); + break; + } + int64_t Kind = MI->getOperand(0).getImm(); + if (!isInt<8>(Kind)) + report("Crash kind must be 8 bit wide", &KindOp, 0); + break; + } case TargetOpcode::G_VECREDUCE_SEQ_FADD: case TargetOpcode::G_VECREDUCE_SEQ_FMUL: { LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index b4b975cce007..b1f514f75207 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -8436,6 +8436,9 @@ def ubsan_trap_xform : SDNodeXFormgetTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32); }]>; +def gi_ubsan_trap_xform : GICustomOperandRenderer<"renderUbsanTrap">, + GISDNodeXFormEquiv; + def ubsan_trap_imm : TImmLeaf(Imm); }], ubsan_trap_xform>; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 677dd0b502b9..a8f2c45279e6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -479,6 +479,8 @@ private: int OpIdx = -1) const; void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx = -1) const; + void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx = -1) const; void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, @@ -6159,16 +6161,6 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects( constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); break; } - case Intrinsic::trap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000); - break; - case Intrinsic::ubsantrap: - MIB.buildInstr(AArch64::BRK, {}, {}) - .addImm(I.getOperand(1).getImm() | ('U' << 8)); - break; case Intrinsic::aarch64_neon_ld1x2: { LLT Ty = MRI.getType(I.getOperand(0).getReg()); unsigned Opc = 0; @@ -7663,6 +7655,14 @@ void AArch64InstructionSelector::renderLogicalImm64( MIB.addImm(Enc); } +void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 && + "Expected G_UBSANTRAP"); + MIB.addImm(MI.getOperand(0).getImm() | ('U' << 8)); +} + void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 90872516dd6d..e55d1de01b4f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2030,6 +2030,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET}) .lower(); + getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom(); + getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE, G_INDEXED_LOAD, G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD, G_INDEXED_STORE}) @@ -2134,6 +2136,10 @@ bool AMDGPULegalizerInfo::legalizeCustom( return legalizeGetFPEnv(MI, MRI, B); case TargetOpcode::G_SET_FPENV: return legalizeSetFPEnv(MI, MRI, B); + case TargetOpcode::G_TRAP: + return legalizeTrap(MI, MRI, B); + case TargetOpcode::G_DEBUGTRAP: + return legalizeDebugTrap(MI, MRI, B); default: return false; } @@ -2925,7 +2931,7 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue( // functions that use local objects. However, if these dead functions are // not eliminated, we don't want a compile time error. Just emit a warning // and a trap, since there should be no callable path here. - B.buildIntrinsic(Intrinsic::trap, ArrayRef()); + B.buildTrap(); B.buildUndef(DstReg); MI.eraseFromParent(); return true; @@ -6618,9 +6624,9 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper, } // TODO: Move to selection -bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI, - MachineRegisterInfo &MRI, - MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { if (!ST.isTrapHandlerEnabled() || ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) return legalizeTrapEndpgm(MI, MRI, B); @@ -6726,8 +6732,9 @@ bool AMDGPULegalizerInfo::legalizeTrapHsa( return true; } -bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic( - MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { // Is non-HSA path or trap-handler disabled? Then, report a warning // accordingly if (!ST.isTrapHandlerEnabled() || @@ -7270,10 +7277,6 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, case Intrinsic::amdgcn_struct_buffer_atomic_fadd_v2bf16: case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd_v2bf16: return legalizeBufferAtomic(MI, B, IntrID); - case Intrinsic::trap: - return legalizeTrapIntrinsic(MI, MRI, B); - case Intrinsic::debugtrap: - return legalizeDebugTrapIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_rsq_clamp: return legalizeRsqClampIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_ds_fadd: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index 9661646fffc9..e5ba84a74a0f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -226,16 +226,16 @@ public: bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const; - bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; - bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override; diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp index 3307e840a2af..8468dd6a2211 100644 --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp @@ -508,16 +508,8 @@ bool MipsLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const { MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; const MipsSubtarget &ST = MI.getMF()->getSubtarget(); - const MipsInstrInfo &TII = *ST.getInstrInfo(); - const MipsRegisterInfo &TRI = *ST.getRegisterInfo(); - const RegisterBankInfo &RBI = *ST.getRegBankInfo(); switch (cast(MI).getIntrinsicID()) { - case Intrinsic::trap: { - MachineInstr *Trap = MIRBuilder.buildInstr(Mips::TRAP); - MI.eraseFromParent(); - return constrainSelectedInstRegOperands(*Trap, TII, TRI, RBI); - } case Intrinsic::vacopy: { MachinePointerInfo MPO; LLT PtrTy = LLT::pointer(0, 32); diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index 5738f86e7e9f..3103992a86c0 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -77,8 +77,6 @@ private: MachineRegisterInfo &MRI) const; bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const; - bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB, - MachineRegisterInfo &MRI) const; void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const; bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB, @@ -686,8 +684,6 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) { return selectSelect(MI, MIB, MRI); case TargetOpcode::G_FCMP: return selectFPCompare(MI, MIB, MRI); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWithSideEffects(MI, MIB, MRI); case TargetOpcode::G_FENCE: { AtomicOrdering FenceOrdering = static_cast(MI.getOperand(0).getImm()); @@ -1255,29 +1251,6 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI, return true; } -bool RISCVInstructionSelector::selectIntrinsicWithSideEffects( - MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const { - assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "Unexpected opcode"); - // Find the intrinsic ID. - unsigned IntrinID = cast(MI).getIntrinsicID(); - - // Select the instruction. - switch (IntrinID) { - default: - return false; - case Intrinsic::trap: - MIB.buildInstr(RISCV::UNIMP, {}, {}); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(RISCV::EBREAK, {}, {}); - break; - } - - MI.eraseFromParent(); - return true; -} - void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const { diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp index 8e0f61a85566..9be3812300af 100644 --- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp +++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp @@ -119,8 +119,6 @@ private: MachineFunction &MF) const; bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; - bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, - MachineFunction &MF) const; // emit insert subreg instruction and insert it before MachineInstr &I bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, @@ -434,8 +432,6 @@ bool X86InstructionSelector::select(MachineInstr &I) { return selectMulDivRem(I, MRI, MF); case TargetOpcode::G_SELECT: return selectSelect(I, MRI, MF); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWSideEffects(I, MRI, MF); } return false; @@ -1834,21 +1830,6 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I, return true; } -bool X86InstructionSelector::selectIntrinsicWSideEffects( - MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { - - assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "unexpected instruction"); - - if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) - return false; - - BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); - - I.eraseFromParent(); - return true; -} - InstructionSelector * llvm::createX86InstructionSelector(const X86TargetMachine &TM, X86Subtarget &Subtarget, diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll index fe9427d2678a..edae903fae84 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll @@ -6,7 +6,7 @@ declare void @llvm.trap() define void @unreachable() { ; CHECK-LABEL: name: unreachable ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP unreachable ret void } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll index 5662de4cbdca..f7550ceb2799 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll @@ -48,7 +48,7 @@ define void @bar() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: $x0 = COPY [[LOAD]](p0) ; CHECK-NEXT: BL @_Unwind_Resume, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP %exn.slot = alloca ptr %ehselector.slot = alloca i32 %1 = invoke i32 @foo(i32 42) to label %continue unwind label %cleanup diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index c9e5f8924f8a..ac3c47c8001d 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -752,6 +752,15 @@ # DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_DEBUGTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_UBSANTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir index ad66fa5623e3..25ecce4dd92b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir @@ -26,7 +26,7 @@ body: | ; CHECK-LABEL: name: foo ; CHECK: BRK 1 ; CHECK: RET_ReallyLR - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP RET_ReallyLR ... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir index f4366fb7888e..b242c68e3b07 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir @@ -26,7 +26,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -48,7 +48,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s8) @@ -80,7 +80,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -102,7 +102,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -134,7 +134,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -165,7 +165,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -206,7 +206,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -228,7 +228,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ANYEXT %6(s16) @@ -261,7 +261,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -284,7 +284,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -317,7 +317,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -340,7 +340,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -377,7 +377,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -410,7 +410,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -512,7 +512,7 @@ body: | ; CHECK-NEXT: bb.2: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.3: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s16) @@ -544,7 +544,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %6(s16) @@ -577,7 +577,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -601,7 +601,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -634,7 +634,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -658,7 +658,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -692,7 +692,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -717,7 +717,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %10:_(s32) = G_ZEXT %8(s8) @@ -783,7 +783,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDO]](s16) @@ -804,7 +804,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %6:_(s32) = G_ANYEXT %4(s16) @@ -839,7 +839,7 @@ body: | ; CHECK-NEXT: RET_ReallyLR implicit $w0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP bb.1: successors: %bb.2(0x7ffff800), %bb.3(0x00000800) liveins: $w0, $w1 @@ -860,6 +860,6 @@ body: | RET_ReallyLR implicit $w0 bb.3: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir index b4bc64812b53..305eca792cfb 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir @@ -24,7 +24,7 @@ body: | bb.0: %0:_(s8) = G_CONSTANT i8 0 %1:_(p1) = G_CONSTANT i64 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.1: G_STORE %0, %1 :: (store 1, addrspace 1) @@ -55,7 +55,7 @@ body: | ; GCN-NEXT: S_ENDPGM 0 bb.0: %0:_(s8) = G_CONSTANT i8 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP %1:_(p1) = G_CONSTANT i64 0 bb.1: diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir similarity index 55% rename from llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir rename to llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir index 64388933fda8..dc99ce8d7a09 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 +# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 --- | declare void @llvm.trap() @@ -9,12 +9,15 @@ --- name: f alignment: 4 +legalized: true +regBankSelected: true +tracksRegLiveness: true body: | bb.1 (%ir-block.0): ; MIPS32-LABEL: name: f ; MIPS32: TRAP - ; MIPS32: RetRA - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; MIPS32-NEXT: RetRA + G_TRAP RetRA ... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir index 11789a030e6f..5f52030fc170 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir @@ -14,7 +14,7 @@ body: | ; CHECK-LABEL: name: test_trap ; CHECK: UNIMP ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP PseudoRET ... @@ -28,7 +28,7 @@ body: | ; CHECK-LABEL: name: test_debugtrap ; CHECK: EBREAK ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.debugtrap) + G_DEBUGTRAP PseudoRET ... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir index ea548c296dca..20b8b671ac5a 100644 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir +++ b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir @@ -23,6 +23,6 @@ body: | bb.1 (%ir-block.0): ; CHECK-LABEL: name: trap ; CHECK: TRAP - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/X86/isel-traps.ll b/llvm/test/CodeGen/X86/isel-traps.ll new file mode 100644 index 000000000000..c207387166a6 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-traps.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X64 +; RUN: llc < %s -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X86 + +declare void @llvm.trap() + +define void @test_trap() { +; ALL-LABEL: test_trap: +; ALL: # %bb.0: +; ALL-NEXT: ud2 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.trap() + ret void +} + +define void @test_debugtrap() { +; ALL-LABEL: test_debugtrap: +; ALL: # %bb.0: +; ALL-NEXT: int3 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.debugtrap() + ret void +} + +define void @test_ubsantrap() { +; ALL-LABEL: test_ubsantrap: +; ALL: # %bb.0: +; ALL-NEXT: ud1l 12(%eax), %eax +; ALL-NEXT: ret{{[l|q]}} + call void @llvm.ubsantrap(i8 12) + ret void +} + +define void @test_ubsantrap_custom() nounwind { +; X64-LABEL: test_ubsantrap_custom: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: movl $42, %edi +; X64-NEXT: callq guide@PLT +; X64-NEXT: popq %rax +; X64-NEXT: retq +; +; GISEL-X64-LABEL: test_ubsantrap_custom: +; GISEL-X64: # %bb.0: +; GISEL-X64-NEXT: pushq %rax +; GISEL-X64-NEXT: movl $42, %edi +; GISEL-X64-NEXT: callq guide +; GISEL-X64-NEXT: popq %rax +; GISEL-X64-NEXT: retq +; +; X86-LABEL: test_ubsantrap_custom: +; X86: # %bb.0: +; X86-NEXT: subl $12, %esp +; X86-NEXT: movl $42, (%esp) +; X86-NEXT: calll guide +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl +; +; GISEL-X86-LABEL: test_ubsantrap_custom: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: subl $12, %esp +; GISEL-X86-NEXT: movl $42, %eax +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: calll guide +; GISEL-X86-NEXT: addl $12, %esp +; GISEL-X86-NEXT: retl + call void @llvm.ubsantrap(i8 42) "trap-func-name"="guide" + ret void +} diff --git a/llvm/test/MachineVerifier/test_g_ubsantrap.mir b/llvm/test/MachineVerifier/test_g_ubsantrap.mir new file mode 100644 index 000000000000..d2b219d8650a --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_ubsantrap.mir @@ -0,0 +1,18 @@ +# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target + +--- +name: test_ubsantrap +tracksRegLiveness: true +liveins: +body: | + bb.0: + + ; CHECK: Crash kind must be 8 bit wide + G_UBSANTRAP 4096 + + ; CHECK: Crash kind must be an immediate + %5:_(s32) = IMPLICIT_DEF + G_UBSANTRAP %5 + +... -- GitLab From f886dfed3ae6cf70827cedc8d8aefde6250a239b Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 13:54:35 +0100 Subject: [PATCH 019/879] [libc++] Don't push and pop extensions diagnostics when using clang modules (#85917) Clang modules take a significant compile time hit when pushing and popping diagnostics. Since all the headers are marked as system headers in the modulemap, we can simply disable this pushing and popping when building with clang modules. --- libcxx/include/__config | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libcxx/include/__config b/libcxx/include/__config index 132cace3d5b2..af44af203e7c 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -838,21 +838,33 @@ typedef __char32_t char32_t; # define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions") #endif +// Clang modules take a significant compile time hit when pushing and popping diagnostics. +// Since all the headers are marked as system headers in the modulemap, we can simply disable this +// pushing and popping when building with clang modules. +# if !__has_feature(modules) +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ + _LIBCPP_DIAGNOSTIC_PUSH \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS _LIBCPP_DIAGNOSTIC_POP +# else +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS +# endif + // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect. // clang-format off -# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") \ +# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std { \ inline namespace _LIBCPP_ABI_NAMESPACE { -# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP +# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_POP_EXTENSION_DIAGNOSTICS # define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD \ inline namespace __fs { namespace filesystem { -- GitLab From 57146daeaaf366050dc913db910fcc2995a3e06d Mon Sep 17 00:00:00 2001 From: Harvin Iriawan <25712785+harviniriawan@users.noreply.github.com> Date: Sat, 23 Mar 2024 12:56:25 +0000 Subject: [PATCH 020/879] [CodeGen] Update for scalable MemoryType in MMO (#70452) Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable TypeSize, the MemoryType created becomes a scalable_vector. 2 MMOs that have scalable memory access can then use the updated BasicAA that understands scalable LocationSize. Original Patch by Harvin Iriawan Co-authored-by: David Green --- llvm/include/llvm/Analysis/MemoryLocation.h | 7 --- llvm/include/llvm/CodeGen/MachineFunction.h | 5 +- llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 34 +++++++++---- llvm/lib/CodeGen/MachineInstr.cpp | 37 ++++++++++---- llvm/lib/CodeGen/MachineOperand.cpp | 13 ++--- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++++++------- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 22 ++++---- .../SelectionDAGAddressAnalysis.cpp | 2 - .../SelectionDAG/SelectionDAGBuilder.cpp | 3 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 5 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++-- llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-- .../alloca-load-store-scalable-array.ll | 36 ++++++------- .../alloca-load-store-scalable-struct.ll | 12 ++--- .../rvv/alloca-load-store-scalable-array.ll | 12 ++--- .../rvv/alloca-load-store-scalable-struct.ll | 8 +-- .../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +-- 17 files changed, 165 insertions(+), 118 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 830eed5d60ee..7d896c44f467 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -297,13 +297,6 @@ public: return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags); } - // Return the exact size if the exact size is known at compiletime, - // otherwise return LocationSize::beforeOrAfterPointer(). - static LocationSize getSizeOrUnknown(const TypeSize &T) { - return T.isScalable() ? LocationSize::beforeOrAfterPointer() - : LocationSize::precise(T.getFixedValue()); - } - MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {} explicit MemoryLocation(const Value *Ptr, LocationSize Size, diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index dfbf7a1e7aae..c2bff2794493 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -1058,8 +1058,9 @@ public: int64_t Offset, LocationSize Size) { return getMachineMemOperand( MMO, Offset, - !Size.hasValue() || Size.isScalable() - ? LLT() + !Size.hasValue() ? LLT() + : Size.isScalable() + ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue()) : LLT::scalar(8 * Size.getValue().getKnownMinValue())); } MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index 9fc8ecd60b03..fb9656c09ca3 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -128,14 +128,14 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1, // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: - if (PtrDiff >= 0 && Size1.hasValue()) { + if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // ========PtrDiff========> IsAlias = !((int64_t)Size1.getValue() <= PtrDiff); return true; } - if (PtrDiff < 0 && Size2.hasValue()) { + if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // =====(-PtrDiff)====> @@ -248,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) || + (MUC1.NumBytes.isScalable() && MUC1.Offset != 0)) + return true; + + const bool BothNotScalable = + !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable(); + // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; - if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) + if (BothNotScalable && + GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) return IsAlias; // The following all rely on MMO0 and MMO1 being valid. @@ -267,12 +277,18 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; - if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0, - MUC0.MMO->getAAInfo()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, - MUC1.MMO->getAAInfo()))) + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); + + if (AA->isNoAlias( + MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()), + MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo()))) return false; } diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index fe2f9ccd33a3..8102bb971ba6 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1306,6 +1306,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, LocationSize WidthB = MMOb->getSize(); bool KnownWidthA = WidthA.hasValue(); bool KnownWidthB = WidthB.hasValue(); + bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable(); const Value *ValA = MMOa->getValue(); const Value *ValB = MMOb->getValue(); @@ -1321,12 +1322,14 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, SameVal = true; } - if (SameVal) { + if (SameVal && BothMMONonScalable) { if (!KnownWidthA || !KnownWidthB) return true; int64_t MaxOffset = std::max(OffsetA, OffsetB); - LocationSize LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB; - return (MinOffset + (int)LowWidth.getValue() > MaxOffset); + int64_t LowWidth = (MinOffset == OffsetA) + ? WidthA.getValue().getKnownMinValue() + : WidthB.getValue().getKnownMinValue(); + return (MinOffset + LowWidth > MaxOffset); } if (!AA) @@ -1338,15 +1341,29 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, assert((OffsetA >= 0) && "Negative MachineMemOperand offset"); assert((OffsetB >= 0) && "Negative MachineMemOperand offset"); - int64_t OverlapA = KnownWidthA ? WidthA.getValue() + OffsetA - MinOffset - : MemoryLocation::UnknownSize; - int64_t OverlapB = KnownWidthB ? WidthB.getValue() + OffsetB - MinOffset - : MemoryLocation::UnknownSize; + // If Scalable Location Size has non-zero offset, Width + Offset does not work + // at the moment + if ((WidthA.isScalable() && OffsetA > 0) || + (WidthB.isScalable() && OffsetB > 0)) + return true; + + int64_t OverlapA = + KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset + : MemoryLocation::UnknownSize; + int64_t OverlapB = + KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset + : MemoryLocation::UnknownSize; + + LocationSize LocA = (WidthA.isScalable() || !KnownWidthA) + ? WidthA + : LocationSize::precise(OverlapA); + LocationSize LocB = (WidthB.isScalable() || !KnownWidthB) + ? WidthB + : LocationSize::precise(OverlapB); return !AA->isNoAlias( - MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), - MemoryLocation(ValB, OverlapB, - UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); + MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), + MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); } bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index 937ca539513a..ace05902d5df 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -1107,12 +1107,13 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F, const MDNode *Ranges, SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) - : MachineMemOperand(ptrinfo, F, - !TS.hasValue() || TS.isScalable() - ? LLT() - : LLT::scalar(8 * TS.getValue().getKnownMinValue()), - BaseAlignment, AAInfo, Ranges, SSID, Ordering, - FailureOrdering) {} + : MachineMemOperand( + ptrinfo, F, + !TS.hasValue() ? LLT() + : TS.isScalable() + ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue()) + : LLT::scalar(8 * TS.getValue().getKnownMinValue()), + BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {} void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) { // The Value and Offset may differ due to CSE. But the flags and size diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index dcd0310734ad..e27a8bb8fdac 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -24200,7 +24200,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) { // TODO: Use "BaseIndexOffset" to make this more effective. SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL); - LocationSize StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); + LocationSize StoreSize = LocationSize::precise(VT.getStoreSize()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO; if (Offset.isScalable()) { @@ -27845,14 +27845,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { : (LSN->getAddressingMode() == ISD::PRE_DEC) ? -1 * C->getSExtValue() : 0; - LocationSize Size = - MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize()); - return {LSN->isVolatile(), - LSN->isAtomic(), - LSN->getBasePtr(), - Offset /*base offset*/, - Size, - LSN->getMemOperand()}; + TypeSize Size = LSN->getMemoryVT().getStoreSize(); + return {LSN->isVolatile(), LSN->isAtomic(), + LSN->getBasePtr(), Offset /*base offset*/, + LocationSize::precise(Size), LSN->getMemOperand()}; } if (const auto *LN = cast(N)) return {false /*isVolatile*/, @@ -27894,6 +27890,13 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() && + MUC0.Offset != 0) || + (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() && + MUC1.Offset != 0)) + return true; // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; @@ -27924,18 +27927,22 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Align OrigAlignment1 = MUC1.MMO->getBaseAlign(); LocationSize Size0 = MUC0.NumBytes; LocationSize Size1 = MUC1.NumBytes; + if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 && - Size0.hasValue() && Size1.hasValue() && Size0 == Size1 && - OrigAlignment0 > Size0.getValue() && - SrcValOffset0 % Size0.getValue() == 0 && - SrcValOffset1 % Size1.getValue() == 0) { + Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() && + !Size1.isScalable() && Size0 == Size1 && + OrigAlignment0 > Size0.getValue().getKnownMinValue() && + SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 && + SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) { int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value(); int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value(); // There is no overlap between these relatively aligned accesses of // similar size. Return no alias. - if ((OffAlign0 + (int64_t)Size0.getValue()) <= OffAlign1 || - (OffAlign1 + (int64_t)Size1.getValue()) <= OffAlign0) + if ((OffAlign0 + static_cast( + Size0.getValue().getKnownMinValue())) <= OffAlign1 || + (OffAlign1 + static_cast( + Size1.getValue().getKnownMinValue())) <= OffAlign0) return false; } @@ -27952,12 +27959,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Size0.hasValue() && Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); if (AA->isNoAlias( - MemoryLocation(MUC0.MMO->getValue(), Overlap0, + MemoryLocation(MUC0.MMO->getValue(), Loc0, UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, + MemoryLocation(MUC1.MMO->getValue(), Loc1, UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes()))) return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e2c07e7cb997..0ab5142ab816 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8406,9 +8406,7 @@ SDValue SelectionDAG::getMemIntrinsicNode( EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags, LocationSize Size, const AAMDNodes &AAInfo) { - if (Size.hasValue() && MemVT.isScalableVector()) - Size = LocationSize::beforeOrAfterPointer(); - else if (Size.hasValue() && !Size.getValue()) + if (Size.hasValue() && !Size.getValue()) Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); @@ -8571,7 +8569,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8692,8 +8690,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); MachineFunction &MF = getMachineFunction(); - LocationSize Size = - MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); + LocationSize Size = LocationSize::precise(Val.getValueType().getStoreSize()); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); return getStore(Chain, dl, Val, Ptr, MMO); @@ -8746,8 +8743,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -8841,7 +8838,7 @@ SDValue SelectionDAG::getLoadVP( if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8994,8 +8991,8 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, IsCompressing); } @@ -11734,10 +11731,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely. - // TODO: Make MachineMemOperands aware of scalable vectors. assert( (!MMO->getType().isValid() || - memvt.getStoreSize().getKnownMinValue() <= MMO->getSize().getValue()) && + TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) && "Size mismatch!"); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 9670c3ac8430..f2ab88851b78 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -106,8 +106,6 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0, int64_t PtrDiff; if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { // If the size of memory access is unknown, do not use it to analysis. - // One example of unknown size memory access is to load/store scalable - // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 84df98b8a613..ae6bd7e938e8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4962,7 +4962,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(AS), MachineMemOperand::MOLoad, - LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), Ranges); + LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), + Ranges); if (!UniformBase) { Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 02943b8a4ab1..d0c5e6b99e9e 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -2687,10 +2687,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth( return false; // The maximum vscale is 16 under AArch64, return the maximal extent for the // vector. - Width = WidthN.isScalable() - ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector / - AArch64::SVEBitsPerBlock - : WidthN.getKnownMinValue(); + Width = LocationSize::precise(WidthN); BaseOps.push_back(BaseOp); return true; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5a2fb0239e0a..5214595485ca 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -10364,9 +10364,15 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op, RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Load->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); SDValue NewLoad = - DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), - Load->getMemOperand()); + DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO); SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL); } @@ -10424,9 +10430,17 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op, const auto [MinVLMAX, MaxVLMAX] = RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && - getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) + getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Store->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(), - Store->getMemOperand()); + MMO); + } SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll index 58299696e78f..d4d803a91cfa 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll @@ -3,7 +3,7 @@ define void @UphPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_p8to15 = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1 ; CHECK: RET_ReallyLR @@ -17,7 +17,7 @@ entry: define void @UpaPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1 ; CHECK: RET_ReallyLR @@ -31,7 +31,7 @@ entry: define void @UplPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_3b = COPY %0 ; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1 ; CHECK: RET_ReallyLR diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll index 9a4e01a29ecb..7244ac949ab8 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll @@ -14,12 +14,12 @@ define void @array_1D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret @@ -81,18 +81,18 @@ define void @array_2D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #5, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #4, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #5, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #4, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] -; CHECK-NEXT: st1d { z5.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z4.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #5, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #4, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #5, mul vl] +; CHECK-NEXT: st1d { z3.d }, p0, [sp, #4, mul vl] +; CHECK-NEXT: st1d { z5.d }, p0, [sp, #3, mul vl] +; CHECK-NEXT: st1d { z4.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #6 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll index 7292d52aaf47..f03a6f018d34 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll @@ -13,12 +13,12 @@ define void @test(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll index 1d025a2f776f..1fe91c721f4d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll @@ -18,15 +18,15 @@ define void @test(ptr %addr) { ; CHECK-NEXT: add a2, a0, a1 ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: slli a2, a1, 1 -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v9, (a3) +; CHECK-NEXT: vl1re64.v v9, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v10, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v9, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v9, (a2) -; CHECK-NEXT: add a1, a0, a1 -; CHECK-NEXT: vs1r.v v8, (a1) -; CHECK-NEXT: vs1r.v v10, (a0) +; CHECK-NEXT: vs1r.v v10, (a2) +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: csrrs a0, vlenb, zero ; CHECK-NEXT: slli a0, a0, 2 ; CHECK-NEXT: add sp, sp, a0 diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll index 64031f8a9359..a9a680d54d58 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll @@ -16,13 +16,13 @@ define @test(ptr %addr, i64 %vl) { ; CHECK-NEXT: sub sp, sp, a2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb ; CHECK-NEXT: csrrs a2, vlenb, zero -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v8, (a3) +; CHECK-NEXT: vl1re64.v v8, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v8, (a2) -; CHECK-NEXT: vs1r.v v9, (a0) +; CHECK-NEXT: vs1r.v v9, (a2) ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll index 0544204cce79..52bd15742ef4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll @@ -16,8 +16,8 @@ define void @vpmerge_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer @@ -37,8 +37,8 @@ define void @vpselect_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer -- GitLab From b68e2eba0bc8dd70b88f4271831139ee9b6ed25c Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 15:28:22 +0100 Subject: [PATCH 021/879] [libc++] Vectorize mismatch (#73255) ``` --------------------------------------------------- Benchmark old new --------------------------------------------------- bm_mismatch/1 0.835 ns 2.37 ns bm_mismatch/2 1.44 ns 2.60 ns bm_mismatch/3 2.06 ns 2.83 ns bm_mismatch/4 2.60 ns 3.29 ns bm_mismatch/5 3.15 ns 3.77 ns bm_mismatch/6 3.82 ns 4.17 ns bm_mismatch/7 4.29 ns 4.52 ns bm_mismatch/8 4.78 ns 4.86 ns bm_mismatch/16 9.06 ns 7.54 ns bm_mismatch/64 31.7 ns 19.1 ns bm_mismatch/512 249 ns 8.16 ns bm_mismatch/4096 1956 ns 44.2 ns bm_mismatch/32768 15498 ns 501 ns bm_mismatch/262144 123965 ns 4479 ns bm_mismatch/1048576 495668 ns 21306 ns bm_mismatch/1 0.710 ns 2.12 ns bm_mismatch/2 1.03 ns 2.66 ns bm_mismatch/3 1.29 ns 3.56 ns bm_mismatch/4 1.68 ns 4.29 ns bm_mismatch/5 1.96 ns 5.18 ns bm_mismatch/6 2.59 ns 5.91 ns bm_mismatch/7 2.86 ns 6.63 ns bm_mismatch/8 3.19 ns 7.33 ns bm_mismatch/16 5.48 ns 13.0 ns bm_mismatch/64 16.6 ns 4.06 ns bm_mismatch/512 130 ns 13.8 ns bm_mismatch/4096 985 ns 93.8 ns bm_mismatch/32768 7846 ns 1002 ns bm_mismatch/262144 63217 ns 10637 ns bm_mismatch/1048576 251782 ns 42471 ns bm_mismatch/1 0.716 ns 1.91 ns bm_mismatch/2 1.21 ns 2.49 ns bm_mismatch/3 1.38 ns 3.46 ns bm_mismatch/4 1.71 ns 4.04 ns bm_mismatch/5 2.00 ns 4.98 ns bm_mismatch/6 2.43 ns 5.67 ns bm_mismatch/7 3.05 ns 6.38 ns bm_mismatch/8 3.22 ns 7.09 ns bm_mismatch/16 5.18 ns 12.8 ns bm_mismatch/64 16.6 ns 5.28 ns bm_mismatch/512 129 ns 25.2 ns bm_mismatch/4096 1009 ns 201 ns bm_mismatch/32768 7776 ns 2144 ns bm_mismatch/262144 62371 ns 20551 ns bm_mismatch/1048576 254750 ns 90097 ns ``` --- libcxx/benchmarks/CMakeLists.txt | 1 + .../benchmarks/algorithms/mismatch.bench.cpp | 31 +++ libcxx/docs/ReleaseNotes/19.rst | 2 + libcxx/include/CMakeLists.txt | 1 + libcxx/include/__algorithm/mismatch.h | 82 ++++++- libcxx/include/__algorithm/simd_utils.h | 123 ++++++++++ libcxx/include/__bit/bit_cast.h | 9 + libcxx/include/__bit/countr.h | 13 +- libcxx/include/libcxx.imp | 1 + libcxx/include/module.modulemap | 6 +- .../mismatch/mismatch.pass.cpp | 214 +++++++++++++----- .../mismatch/mismatch_pred.pass.cpp | 119 ---------- 12 files changed, 413 insertions(+), 189 deletions(-) create mode 100644 libcxx/benchmarks/algorithms/mismatch.bench.cpp create mode 100644 libcxx/include/__algorithm/simd_utils.h delete mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt index 3dec6faea13a..387e013afeb6 100644 --- a/libcxx/benchmarks/CMakeLists.txt +++ b/libcxx/benchmarks/CMakeLists.txt @@ -183,6 +183,7 @@ set(BENCHMARK_TESTS algorithms/make_heap_then_sort_heap.bench.cpp algorithms/min.bench.cpp algorithms/min_max_element.bench.cpp + algorithms/mismatch.bench.cpp algorithms/pop_heap.bench.cpp algorithms/pstl.stable_sort.bench.cpp algorithms/push_heap.bench.cpp diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp new file mode 100644 index 000000000000..9274932a764c --- /dev/null +++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +// TODO: Look into benchmarking aligned and unaligned memory explicitly +// (currently things happen to be aligned because they are malloced that way) +template +static void bm_mismatch(benchmark::State& state) { + std::vector vec1(state.range(), '1'); + std::vector vec2(state.range(), '1'); + std::mt19937_64 rng(std::random_device{}()); + + vec1.back() = '2'; + for (auto _ : state) { + benchmark::DoNotOptimize(vec1); + benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin())); + } +} +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); + +BENCHMARK_MAIN(); diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index cac42f9c3c3f..dd39c1bbbc78 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -51,6 +51,8 @@ Improvements and New Features - The performance of growing ``std::vector`` has been improved for trivially relocatable types. - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector::iterator``\s, resulting in a performance increase of up to 1400x. +- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance + improvements. Deprecations and Removals ------------------------- diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 6ed8d21d98a1..982b85e4e2d6 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -217,6 +217,7 @@ set(files __algorithm/shift_right.h __algorithm/shuffle.h __algorithm/sift_down.h + __algorithm/simd_utils.h __algorithm/sort.h __algorithm/sort_heap.h __algorithm/stable_partition.h diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index d345b6048a7e..4eb693a1f2e9 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -11,23 +11,93 @@ #define _LIBCPP___ALGORITHM_MISMATCH_H #include <__algorithm/comp.h> +#include <__algorithm/simd_utils.h> +#include <__algorithm/unwrap_iter.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__functional/identity.h> +#include <__type_traits/invoke.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_equality_comparable.h> +#include <__type_traits/operation_traits.h> +#include <__utility/move.h> #include <__utility/pair.h> +#include <__utility/unreachable.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + while (__first1 != __last1) { + if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) + break; + ++__first1; + ++__first2; + } + return std::make_pair(std::move(__first1), std::move(__first2)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#if _LIBCPP_VECTORIZE_ALGORITHMS + +template ::value && __desugars_to<__equal_tag, _Pred, _Tp, _Tp>::value && + __is_identity<_Proj1>::value && __is_identity<_Proj2>::value, + int> = 0> +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*> +__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + constexpr size_t __unroll_count = 4; + constexpr size_t __vec_size = __native_vector_size<_Tp>; + using __vec = __simd_vector<_Tp, __vec_size>; + if (!__libcpp_is_constant_evaluated()) { + while (static_cast(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] { + __vec __lhs[__unroll_count]; + __vec __rhs[__unroll_count]; + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + __lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size); + __rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size); + } + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) { + auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res); + return {__first1 + __offset, __first2 + __offset}; + } + } + + __first1 += __unroll_count * __vec_size; + __first2 += __unroll_count * __vec_size; + } + } + // TODO: Consider vectorizing the tail + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#endif // _LIBCPP_VECTORIZE_ALGORITHMS + template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) { - for (; __first1 != __last1; ++__first1, (void)++__first2) - if (!__pred(*__first1, *__first2)) - break; - return pair<_InputIterator1, _InputIterator2>(__first1, __first2); + __identity __proj; + auto __res = std::__mismatch( + std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj); + return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second)); } template @@ -59,4 +129,6 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP___ALGORITHM_MISMATCH_H diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h new file mode 100644 index 000000000000..1aedb3db010f --- /dev/null +++ b/libcxx/include/__algorithm/simd_utils.h @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H +#define _LIBCPP___ALGORITHM_SIMD_UTILS_H + +#include <__bit/bit_cast.h> +#include <__bit/countr.h> +#include <__config> +#include <__type_traits/is_arithmetic.h> +#include <__type_traits/is_same.h> +#include <__utility/integer_sequence.h> +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// TODO: Find out how altivec changes things and allow vectorizations there too. +#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__) +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1 +#else +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) +# define _LIBCPP_VECTORIZE_ALGORITHMS 1 +#else +# define _LIBCPP_VECTORIZE_ALGORITHMS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +_LIBCPP_BEGIN_NAMESPACE_STD + +// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance +// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms +// as far as benchmarks are concerned. +# if defined(__AVX__) +template +inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); +# elif defined(__SSE__) || defined(__ARM_NEON__) +template +inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); +# elif defined(__MMX__) +template +inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); +# else +template +inline constexpr size_t __native_vector_size = 1; +# endif + +template +using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; + +template +inline constexpr size_t __simd_vector_size_v = []() -> size_t { + static_assert(_False, "Not a vector!"); +}(); + +template +inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; + +template +_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { + return _Tp{}; +} + +template +using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); + +// This isn't inlined without always_inline when loading chars. +template +_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept { + return [=](index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { + return _VecT{__ptr[_Indices]...}; + }(make_index_sequence<__simd_vector_size_v<_VecT>>{}); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { + return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { + using __mask_vec = __simd_vector; + + // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 + auto __impl = [&](_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { + return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))); + }; + + if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { + return __impl(uint8_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { + return __impl(uint16_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { + return __impl(uint32_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { + return __impl(uint64_t{}); + } else { + static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); + return 0; + } +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { + return std::__find_first_set(~__vec); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H diff --git a/libcxx/include/__bit/bit_cast.h b/libcxx/include/__bit/bit_cast.h index f20b39ae748b..6298810f3733 100644 --- a/libcxx/include/__bit/bit_cast.h +++ b/libcxx/include/__bit/bit_cast.h @@ -19,6 +19,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD +#ifndef _LIBCPP_CXX03_LANG + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept { + return __builtin_bit_cast(_ToType, __from); +} + +#endif // _LIBCPP_CXX03_LANG + #if _LIBCPP_STD_VER >= 20 template diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h index 0cc679f87a99..b6b3ac52ca4e 100644 --- a/libcxx/include/__bit/countr.h +++ b/libcxx/include/__bit/countr.h @@ -35,10 +35,8 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ct return __builtin_ctzll(__x); } -#if _LIBCPP_STD_VER >= 20 - -template <__libcpp_unsigned_integer _Tp> -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT { if (__t == 0) return numeric_limits<_Tp>::digits; @@ -59,6 +57,13 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n } } +#if _LIBCPP_STD_VER >= 20 + +template <__libcpp_unsigned_integer _Tp> +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { + return std::__countr_zero(__t); +} + template <__libcpp_unsigned_integer _Tp> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept { return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits; diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index 77b7befd44f5..56ea58262828 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -217,6 +217,7 @@ { include: [ "<__algorithm/shift_right.h>", "private", "", "public" ] }, { include: [ "<__algorithm/shuffle.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sift_down.h>", "private", "", "public" ] }, + { include: [ "<__algorithm/simd_utils.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort_heap.h>", "private", "", "public" ] }, { include: [ "<__algorithm/stable_partition.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index f36a47cef009..03d18775631e 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -697,7 +697,10 @@ module std_private_algorithm_minmax [system export * } module std_private_algorithm_minmax_element [system] { header "__algorithm/minmax_element.h" } -module std_private_algorithm_mismatch [system] { header "__algorithm/mismatch.h" } +module std_private_algorithm_mismatch [system] { + header "__algorithm/mismatch.h" + export std_private_algorithm_simd_utils +} module std_private_algorithm_move [system] { header "__algorithm/move.h" } module std_private_algorithm_move_backward [system] { header "__algorithm/move_backward.h" } module std_private_algorithm_next_permutation [system] { header "__algorithm/next_permutation.h" } @@ -1048,6 +1051,7 @@ module std_private_algorithm_sort [system header "__algorithm/sort.h" export std_private_debug_utils_strict_weak_ordering_check } +module std_private_algorithm_simd_utils [system] { header "__algorithm/simd_utils.h" } module std_private_algorithm_sort_heap [system] { header "__algorithm/sort_heap.h" } module std_private_algorithm_stable_partition [system] { header "__algorithm/stable_partition.h" } module std_private_algorithm_stable_sort [system] { header "__algorithm/stable_sort.h" } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp index cc588c095ccf..e7f3994d977d 100644 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp @@ -16,79 +16,173 @@ // template // constexpr pair // constexpr after c++17 // mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2); // C++14 +// +// template Pred> +// requires CopyConstructible +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); +// +// template +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 + +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=50000000 +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-ops-limit): -fconstexpr-ops-limit=100000000 #include +#include #include +#include #include "test_macros.h" #include "test_iterators.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic)); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic)); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic)); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic)); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic))); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic))); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } +#include "type_algorithms.h" + +template +TEST_CONSTEXPR_CXX20 void check(Container1 lhs, Container2 rhs, size_t offset) { + if (lhs.size() == rhs.size()) { + assert(std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + } + +#if TEST_STD_VER >= 14 + assert( + std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data()), Iter(rhs.data() + rhs.size())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + Iter(rhs.data() + rhs.size()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); #endif +} -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib)) - == (std::pair(II(ia+3), II(ib+3)))); - - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - -#if TEST_STD_VER > 11 // We have the four iteration version - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+sb)) - == (std::pair(II(ia+3), II(ib+3)))); +struct NonTrivial { + int i_; + + TEST_CONSTEXPR_CXX20 NonTrivial(int i) : i_(i) {} + TEST_CONSTEXPR_CXX20 NonTrivial(NonTrivial&& other) : i_(other.i_) { other.i_ = 0; } + + TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivial& lhs, const NonTrivial& rhs) { return lhs.i_ == rhs.i_; } +}; + +struct ModTwoComp { + TEST_CONSTEXPR_CXX20 bool operator()(int lhs, int rhs) { return lhs % 2 == rhs % 2; } +}; + +template +TEST_CONSTEXPR_CXX20 bool test() { + { // empty ranges + std::array lhs = {}; + std::array rhs = {}; + check(lhs, rhs, 0); + } + + { // same range without mismatch + std::array lhs = {0, 1, 2, 3, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 8); + } + + { // same range with mismatch + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 3); + } + + { // second range is smaller + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1}; + check(lhs, rhs, 2); + } + + { // first range is smaller + std::array lhs = {0, 1}; + std::array rhs = {0, 1, 2, 2, 0, 1, 2, 3}; + check(lhs, rhs, 2); + } + + { // use a custom comparator + std::array lhs = {0, 2, 3, 4}; + std::array rhs = {0, 0, 4, 4}; + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#if TEST_STD_VER >= 14 + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), rhs.data() + rhs.size(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#endif + } - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib+sb)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); + return true; +} +struct Test { + template + TEST_CONSTEXPR_CXX20 void operator()() { + test(); + } +}; + +TEST_CONSTEXPR_CXX20 bool test() { + types::for_each(types::cpp17_input_iterator_list(), Test()); + + { // use a non-integer type to also test the general case - all elements match + std::array lhs = {1, 2, 3, 4, 5, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 8); + } + + { // use a non-integer type to also test the general case - not all elements match + std::array lhs = {1, 2, 3, 4, 7, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 4); + } + + return true; +} - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+2)) - == (std::pair(II(ia+2), II(ib+2)))); +int main(int, char**) { + test(); +#if TEST_STD_VER >= 20 + static_assert(test()); #endif -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif + { // check with a lot of elements to test the vectorization optimization + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + } return 0; } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp deleted file mode 100644 index bda4ec7ba5ed..000000000000 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template Pred> -// requires CopyConstructible -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); -// -// template -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 - -#include -#include -#include - -#include "test_macros.h" -#include "test_iterators.h" -#include "counting_predicates.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool eq(int a, int b) { return a == b; } - -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), eq); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic), eq); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), eq); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic), eq); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic)), eq); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic)), eq); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } -#endif - - -#if TEST_STD_VER > 11 -#define HAS_FOUR_ITERATOR_VERSION -#endif - -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - typedef std::equal_to EQ; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - binary_counting_predicate bcp((EQ())); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), std::ref(bcp)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < sa); - bcp.reset(); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib + sb), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), std::ref(bcp)) - == (std::pair(II(ia+3), II(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < std::min(sa, sb)); -#endif - - assert(std::mismatch(ia, ia + sa, ib, EQ()) == - (std::pair(ia+3,ib+3))); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(ia, ia + sa, ib, ib + sb, EQ()) == - (std::pair(ia+3,ib+3))); - assert(std::mismatch(ia, ia + sa, ib, ib + 2, EQ()) == - (std::pair(ia+2,ib+2))); -#endif - -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif - - return 0; -} -- GitLab From 3f5e649ff64a92a732027ef76e33a1c8b1722d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 16:46:32 +0100 Subject: [PATCH 022/879] [Libomptarget] Fix linking to LLVM dylib (#86397) Use `LINK_COMPONENTS` parameter of `add_llvm_library` rather than passing LLVM components directly to `target_link_libraries`, in order to ensure that LLVM dylib is linked correctly when used. Otherwise, CMake insists on linking to static libraries that aren't present on distributions doing pure dylib installs, such as Gentoo. This fixes a regression introduced in dcbddc25250158469c5635ad2ae4095faef53dfd. --- .../plugins-nextgen/CMakeLists.txt | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index c19fd80592d6..dbd82ac94517 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -14,41 +14,40 @@ set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) function(add_target_library target_name lib_name) - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) - add_llvm_library(${target_name} SHARED + LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + AggressiveInstCombine + Analysis + BinaryFormat + BitReader + BitWriter + CodeGen + Core + Extensions + InstCombine + Instrumentation + IPO + IRReader + Linker + MC + Object + Passes + Remarks + ScalarOpts + Support + Target + TargetParser + TransformUtils + Vectorize + NO_INSTALL_RPATH BUILDTREE_ONLY ) llvm_update_compile_flags(${target_name}) target_link_libraries(${target_name} PRIVATE - PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) + PluginCommon ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) target_compile_definitions(${target_name} PRIVATE -- GitLab From d7ce6b4d96c8879f38ba4cb5fdb1cc09d5b129e5 Mon Sep 17 00:00:00 2001 From: Mike Rice Date: Sat, 23 Mar 2024 09:53:29 -0700 Subject: [PATCH 023/879] [clang-tidy] Fix result check after overwriteChangedFiles() (#86360) If any return from overwriteChangedFiles is true some fixes were not applied. --- clang-tools-extra/clang-tidy/ClangTidy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp index 40ac6918faf4..b877ea06dc05 100644 --- a/clang-tools-extra/clang-tidy/ClangTidy.cpp +++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp @@ -233,7 +233,7 @@ public: if (!tooling::applyAllReplacements(Replacements.get(), Rewrite)) { llvm::errs() << "Can't apply replacements for file " << File << "\n"; } - AnyNotWritten &= Rewrite.overwriteChangedFiles(); + AnyNotWritten |= Rewrite.overwriteChangedFiles(); } if (AnyNotWritten) { -- GitLab From 87c7f4a12b2a1b723a78d760761ee473f52c4cee Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:03:09 -0700 Subject: [PATCH 024/879] [MC] Remove unnecessary reversal of relocations. NFC Commit f44db24e1fd948c75c87aea017646f16553d3361 (2015) enabled this simplication. --- llvm/lib/MC/ELFObjectWriter.cpp | 9 +-------- .../lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 5 ++--- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index 3c4d3ab9a508..a6fb6b5c1a4e 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -939,18 +939,11 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec) { std::vector &Relocs = OWriter.Relocations[&Sec]; - // We record relocations by pushing to the end of a vector. Reverse the vector - // to get the relocations in the order they were created. - // In most cases that is not important, but it can be for special sections - // (.eh_frame) or specific relocations (TLS optimizations on SystemZ). - std::reverse(Relocs.begin(), Relocs.end()); - // Sort the relocation entries. MIPS needs this. OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { - const ELFRelocationEntry &Entry = Relocs[e - i - 1]; + for (const ELFRelocationEntry &Entry : Relocs) { unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; if (is64Bit()) { diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp index 181b82f14bfe..4d6a00c14a35 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp @@ -498,10 +498,9 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm, assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed"); - // Overwrite the original vector with the sorted elements. The caller expects - // them in reverse order. + // Overwrite the original vector with the sorted elements. unsigned CopyTo = 0; - for (const auto &R : reverse(Sorted)) + for (const auto &R : Sorted) Relocs[CopyTo++] = R.R; } -- GitLab From 3a63f737e29a2382c8ec26c0d360bd77c01a27a1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:15:47 -0700 Subject: [PATCH 025/879] [MC] Refactor writeRelocations. NFC MIPS is different and should better off use separate code. --- llvm/lib/MC/ELFObjectWriter.cpp | 60 ++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index a6fb6b5c1a4e..f4c6cbc8dd44 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -943,40 +943,28 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (const ELFRelocationEntry &Entry : Relocs) { - unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; - - if (is64Bit()) { - write(Entry.Offset); - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { - write(uint32_t(Index)); - + if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + write(uint32_t(Symidx)); write(OWriter.TargetObjectWriter->getRSsym(Entry.Type)); write(OWriter.TargetObjectWriter->getRType3(Entry.Type)); write(OWriter.TargetObjectWriter->getRType2(Entry.Type)); write(OWriter.TargetObjectWriter->getRType(Entry.Type)); + if (Rela) + write(Entry.Addend); } else { - struct ELF::Elf64_Rela ERE64; - ERE64.setSymbolAndType(Index, Entry.Type); - write(ERE64.r_info); - } - if (Rela) - write(Entry.Addend); - } else { - write(uint32_t(Entry.Offset)); - - struct ELF::Elf32_Rela ERE32; - ERE32.setSymbolAndType(Index, Entry.Type); - write(ERE32.r_info); - - if (Rela) - write(uint32_t(Entry.Addend)); - - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE32; + ERE32.setSymbolAndType(Symidx, Entry.Type); + write(ERE32.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); if (uint32_t RType = OWriter.TargetObjectWriter->getRType2(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); @@ -984,13 +972,31 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, if (uint32_t RType = OWriter.TargetObjectWriter->getRType3(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); } } } + return; + } + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + ELF::Elf64_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(Entry.Addend); + } else { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); + } } } -- GitLab From 39c8e87717fbc611b9e84f62edf656608ae52e5c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 23 Mar 2024 18:43:14 +0100 Subject: [PATCH 026/879] [VPlan] Move recording of Inst->VPValue to VPRecipeBuilder (NFCI). (#84464) Instead of keeping a mapping of Inst->VPValues (of their corresponding recipes) in VPlan's Value2VPValue mapping, keep it in VPRecipeBuilder instead. After recently replacing the last user of this mapping after initial construction, this mapping is only needed for recipe construction (to map IR operands to VPValue operands). By moving the mapping, VPlan's VPValue tracking can be simplified and limited only to live-ins. It also allows removing disableValue2VPValue and associated machinery & asserts. PR: https://github.com/llvm/llvm-project/pull/84464 --- .../Transforms/Vectorize/LoopVectorize.cpp | 47 +++++++------------ .../Transforms/Vectorize/VPRecipeBuilder.h | 35 +++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 25 ++-------- .../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++--- 4 files changed, 46 insertions(+), 76 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2163930b02c1..28748142ae92 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7898,6 +7898,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, } } +iterator_range>> +VPRecipeBuilder::mapToVPValues(User::op_range Operands) { + std::function Fn = [this](Value *Op) { + if (auto *I = dyn_cast(Op)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(Op); + }; + return map_range(Operands, Fn); +} + VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { assert(is_contained(predecessors(Dst), Src) && "Invalid edge"); @@ -7922,7 +7934,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { if (OrigLoop->isLoopExiting(Src)) return EdgeMaskCache[Edge] = SrcMask; - VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition()); + VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan); assert(EdgeMask && "No Edge Mask found for condition"); if (BI->getSuccessor(0) != Dst) @@ -8383,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I, BlockInMask = getBlockInMask(I->getParent()); } - auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()), + auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()), IsUniform, BlockInMask); return Recipe; } @@ -8399,10 +8411,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, if (Phi->getParent() != OrigLoop->getHeader()) return tryToBlend(Phi, Operands); - // Always record recipes for header phis. Later first-order recurrence phis - // can have earlier phis as incoming values. - recordRecipeOf(Phi); - if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range))) return Recipe; @@ -8427,14 +8435,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } - // Record the incoming value from the backedge, so we can add the incoming - // value from the backedge after all recipes have been created. - auto *Inc = cast( - Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); - auto RecipeIter = Ingredient2Recipe.find(Inc); - if (RecipeIter == Ingredient2Recipe.end()) - recordRecipeOf(Inc); - PhisToFix.push_back(PhiRecipe); return PhiRecipe; } @@ -8522,7 +8522,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the // original exit block. static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, - VPlan &Plan) { + VPRecipeBuilder &Builder, VPlan &Plan) { BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock(); BasicBlock *ExitingBB = OrigLoop->getExitingBlock(); // Only handle single-exit loops with unique exit blocks for now. @@ -8533,7 +8533,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, for (PHINode &ExitPhi : ExitBB->phis()) { Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB); - VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue); + VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan); Plan.addLiveOut(&ExitPhi, V); } } @@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!getDecisionAndClampRange(applyIG, Range)) continue; InterleaveGroups.insert(IG); - for (unsigned i = 0; i < IG->getFactor(); i++) - if (Instruction *Member = IG->getMember(i)) - RecipeBuilder.recordRecipeOf(Member); }; // --------------------------------------------------------------------------- @@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Operands.push_back(Plan->getVPValueOrAddLiveIn( Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); } else { - auto OpRange = Plan->mapToVPValues(Instr->operands()); + auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands()); Operands = {OpRange.begin(), OpRange.end()}; } @@ -8662,10 +8659,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB); if (!Recipe) Recipe = RecipeBuilder.handleReplication(Instr, Range); - for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); - } RecipeBuilder.setRecipe(Instr, Recipe); if (isa(Recipe)) { @@ -8697,7 +8690,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // and there is nothing to fix from vector loop; phis should have incoming // from scalar loop only. } else - addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan); + addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan); assert(isa(Plan->getVectorLoopRegion()) && !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && @@ -8765,10 +8758,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV); } - // From this point onwards, VPlan-to-VPlan transformations may change the plan - // in ways that accessing values using original IR values is incorrect. - Plan->disableValue2VPValue(); - VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) { return Legal->blockNeedsPredication(BB); }); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 29a395c35731..fbb70f333685 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -52,9 +52,8 @@ class VPRecipeBuilder { EdgeMaskCacheTy EdgeMaskCache; BlockMaskCacheTy BlockMaskCache; - // VPlan-VPlan transformations support: Hold a mapping from ingredients to - // their recipe. To save on memory, only do so for selected ingredients, - // marked by having a nullptr entry in this map. + // VPlan construction support: Hold a mapping from ingredients to + // their recipe. DenseMap Ingredient2Recipe; /// Cross-iteration reduction & first-order recurrence phis for which we need @@ -117,13 +116,10 @@ public: ArrayRef Operands, VFRange &Range, VPBasicBlock *VPBB); - /// Set the recipe created for given ingredient. This operation is a no-op for - /// ingredients that were not marked using a nullptr entry in the map. + /// Set the recipe created for given ingredient. void setRecipe(Instruction *I, VPRecipeBase *R) { - if (!Ingredient2Recipe.count(I)) - return; - assert(Ingredient2Recipe[I] == nullptr && - "Recipe already set for ingredient"); + assert(!Ingredient2Recipe.contains(I) && + "Cannot reset recipe for instruction."); Ingredient2Recipe[I] = R; } @@ -146,14 +142,6 @@ public: /// between SRC and DST. VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const; - /// Mark given ingredient for recording its recipe once one is created for - /// it. - void recordRecipeOf(Instruction *I) { - assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) && - "Recipe already set for ingredient"); - Ingredient2Recipe[I] = nullptr; - } - /// Return the recipe created for given ingredient. VPRecipeBase *getRecipe(Instruction *I) { assert(Ingredient2Recipe.count(I) && @@ -171,6 +159,19 @@ public: /// Add the incoming values from the backedge to reduction & first-order /// recurrence cross-iteration phis. void fixHeaderPhis(); + + /// Returns a range mapping the values of the range \p Operands to their + /// corresponding VPValues. + iterator_range>> + mapToVPValues(User::op_range Operands); + + VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) { + if (auto *I = dyn_cast(V)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(V); + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d77c7554d50e..22173954f7ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2872,10 +2872,6 @@ class VPlan { /// definitions are VPValues that hold a pointer to their underlying IR. SmallVector VPLiveInsToFree; - /// Indicates whether it is safe use the Value2VPValue mapping or if the - /// mapping cannot be used any longer, because it is stale. - bool Value2VPValueEnabled = true; - /// Values used outside the plan. MapVector LiveOuts; @@ -2954,10 +2950,6 @@ public: /// Returns VF * UF of the vector loop region. VPValue &getVFxUF() { return VFxUF; } - /// Mark the plan to indicate that using Value2VPValue is not safe any - /// longer, because it may be stale. - void disableValue2VPValue() { Value2VPValueEnabled = false; } - void addVF(ElementCount VF) { VFs.insert(VF); } void setVF(ElementCount VF) { @@ -2987,8 +2979,7 @@ public: void setName(const Twine &newName) { Name = newName.str(); } void addVPValue(Value *V, VPValue *VPV) { - assert((Value2VPValueEnabled || VPV->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(VPV->isLiveIn() && "VPV must be a live-in."); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); Value2VPValue[V] = VPV; @@ -2998,8 +2989,8 @@ public: VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(Value2VPValue[V]->isLiveIn() && + "Only live-ins should be in mapping"); return Value2VPValue[V]; } @@ -3030,16 +3021,6 @@ public: LLVM_DUMP_METHOD void dump() const; #endif - /// Returns a range mapping the values the range \p Operands to their - /// corresponding VPValues. - iterator_range>> - mapToVPValues(User::op_range Operands) { - std::function Fn = [this](Value *Op) { - return getVPValueOrAddLiveIn(Op); - }; - return map_range(Operands, Fn); - } - /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { return cast(getEntry()->getSingleSuccessor()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a91ccefe4b6d..00cec1148528 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -48,15 +48,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes( VPRecipeBase *NewRecipe = nullptr; if (auto *VPPhi = dyn_cast(&Ingredient)) { auto *Phi = cast(VPPhi->getUnderlyingValue()); - if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) { - VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); - VPValue *Step = - vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); - } else { - Plan->addVPValue(Phi, VPPhi); + const auto *II = GetIntOrFpInductionDescriptor(Phi); + if (!II) continue; - } + + VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); + VPValue *Step = + vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); } else { assert(isa(&Ingredient) && "only VPInstructions expected here"); -- GitLab From e14c6fa31a303312d7561aa0c7219d687f3f2c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 20:26:20 +0100 Subject: [PATCH 027/879] [clang] [cmake] Add cmake module dir before using GetDarwinLinkerVersion (#86386) Move the code adding top-level cmake/Modules directory to CMAKE_MODULE_PATH prior to including `GetDarwinLinkerVersion`, in order to fix standalone builds. Fixes a regression introduced by 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b. --- clang/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index ee783d52e4a4..284b2af24dda 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -13,6 +13,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(CLANG_BUILT_STANDALONE TRUE) endif() +# Make sure that our source directory is on the current cmake module path so that +# we can include cmake files from this directory. +list(INSERT CMAKE_MODULE_PATH 0 + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" + "${LLVM_COMMON_CMAKE_UTILS}/Modules" + ) + # Must go below project(..) include(GNUInstallDirs) include(GetDarwinLinkerVersion) @@ -141,13 +148,6 @@ if(CLANG_BUILT_STANDALONE) endif() # LLVM_INCLUDE_TESTS endif() # standalone -# Make sure that our source directory is on the current cmake module path so that -# we can include cmake files from this directory. -list(INSERT CMAKE_MODULE_PATH 0 - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" - "${LLVM_COMMON_CMAKE_UTILS}/Modules" - ) - # This allows disabling clang's XML dependency even if LLVM finds libxml2. # By default, clang depends on libxml2 if LLVM does. option(CLANG_ENABLE_LIBXML2 "Whether libclang may depend on libxml2" -- GitLab From a91cd53de34fcde469d5d02e84554a06798ad2b1 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 16:08:31 -0700 Subject: [PATCH 028/879] [BOLT][NFC] Refactor BAT metadata data structures Hide the implementations of `FuncHashes` and `BBHashMap` classes, getting rid of `at` accessors that could throw an exception. Test Plan: NFC Reviewers: ayermolo, maksfb, dcci, rafaelauler Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86353 --- .../bolt/Profile/BoltAddressTranslation.h | 117 +++++++++++++++--- bolt/lib/Profile/BoltAddressTranslation.cpp | 85 ++++++------- 2 files changed, 137 insertions(+), 65 deletions(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index d583ce0b76a2..f8c35f8066f7 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -115,17 +115,6 @@ public: /// Save function and basic block hashes used for metadata dump. void saveMetadata(BinaryContext &BC); - /// Returns BB hash by function output address (after BOLT) and basic block - /// input offset. - size_t getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - - /// Returns BF hash by function output address (after BOLT). - size_t getBFHash(uint64_t OutputAddress) const; - - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -135,7 +124,7 @@ private: /// emitted for the start of the BB. More entries may be emitted to cover /// the location of calls or any instruction that may change control flow. void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress); + uint64_t FuncInputAddress, uint64_t FuncOutputAddress); /// Write the serialized address translation table for a function. template @@ -158,10 +147,6 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; - std::unordered_map> FuncHashes; - /// Map a function to its basic blocks count std::unordered_map NumBasicBlocksMap; @@ -174,6 +159,106 @@ private: /// Identifies the address of a control-flow changing instructions in a /// translation map entry const static uint32_t BRANCHENTRY = 0x1; + +public: + /// Map basic block input offset to a basic block index and hash pair. + class BBHashMapTy { + class EntryTy { + unsigned Index; + size_t Hash; + + public: + unsigned getBBIndex() const { return Index; } + size_t getBBHash() const { return Hash; } + EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint32_t BBInputOffset) const { + auto It = Map.find(BBInputOffset); + assert(It != Map.end()); + return It->second; + } + + public: + bool isInputBlock(uint32_t InputOffset) const { + return Map.count(InputOffset); + } + + unsigned getBBIndex(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBIndex(); + } + + size_t getBBHash(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBHash(); + } + + void addEntry(uint32_t BBInputOffset, unsigned BBIndex, size_t BBHash) { + Map.emplace(BBInputOffset, EntryTy(BBIndex, BBHash)); + } + + size_t getNumBasicBlocks() const { return Map.size(); } + }; + + /// Map function output address to its hash and basic blocks hash map. + class FuncHashesTy { + class EntryTy { + size_t Hash; + BBHashMapTy BBHashMap; + + public: + size_t getBFHash() const { return Hash; } + const BBHashMapTy &getBBHashMap() const { return BBHashMap; } + EntryTy(size_t Hash) : Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint64_t FuncOutputAddress) const { + auto It = Map.find(FuncOutputAddress); + assert(It != Map.end()); + return It->second; + } + + public: + size_t getBFHash(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBFHash(); + } + + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBBHashMap(); + } + + void addEntry(uint64_t FuncOutputAddress, size_t BFHash) { + Map.emplace(FuncOutputAddress, EntryTy(BFHash)); + } + + size_t getNumFunctions() const { return Map.size(); }; + + size_t getNumBasicBlocks() const { + size_t NumBasicBlocks{0}; + for (auto &I : Map) + NumBasicBlocks += I.second.getBBHashMap().getNumBasicBlocks(); + return NumBasicBlocks; + } + }; + + /// Returns BF hash by function output address (after BOLT). + size_t getBFHash(uint64_t FuncOutputAddress) const { + return FuncHashes.getBFHash(FuncOutputAddress); + } + + /// Returns BBHashMap by function output address (after BOLT). + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return FuncHashes.getBBHashMap(FuncOutputAddress); + } + + BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) { + return const_cast( + std::as_const(*this).getBBHashMap(FuncOutputAddress)); + } + +private: + FuncHashesTy FuncHashes; }; } // namespace bolt diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 31886f4c8025..57d2aff71977 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -22,12 +22,10 @@ const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress) { - uint64_t HotFuncAddress = ColdPartSource.count(FuncAddress) - ? ColdPartSource[FuncAddress] - : FuncAddress; + uint64_t FuncInputAddress, + uint64_t FuncOutputAddress) { const uint64_t BBOutputOffset = - BB.getOutputAddressRange().first - FuncAddress; + BB.getOutputAddressRange().first - FuncOutputAddress; const uint32_t BBInputOffset = BB.getInputOffset(); // Every output BB must track back to an input BB for profile collection @@ -42,11 +40,14 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << "BB " << BB.getName() << "\n"); LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(BBOutputOffset) << " Val: " << Twine::utohexstr(BBInputOffset) << "\n"); - LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", - getBBHash(HotFuncAddress, BBInputOffset))); - (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); + // NB: in `writeEntriesForBB` we use the input address because hashes are + // saved early in `saveMetadata` before output addresses are assigned. + const BBHashMapTy &BBHashMap = getBBHashMap(FuncInputAddress); + (void)BBHashMap; + LLVM_DEBUG( + dbgs() << formatv(" Hash: {0:x}\n", BBHashMap.getBBHash(BBInputOffset))); + LLVM_DEBUG( + dbgs() << formatv(" Index: {0}\n", BBHashMap.getBBIndex(BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -63,7 +64,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const auto InputAddress = BB.getFunction()->getAddress() + InputOffset; const auto OutputAddress = IOAddressMap.lookup(InputAddress); assert(OutputAddress && "Unknown instruction address"); - const auto OutputOffset = *OutputAddress - FuncAddress; + const auto OutputOffset = *OutputAddress - FuncOutputAddress; // Is this the first instruction in the BB? No need to duplicate the entry. if (OutputOffset == BBOutputOffset) @@ -99,7 +100,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { MapTy Map; for (const BinaryBasicBlock *const BB : Function.getLayout().getMainFragment()) - writeEntriesForBB(Map, *BB, Function.getOutputAddress()); + writeEntriesForBB(Map, *BB, InputAddress, OutputAddress); Maps.emplace(Function.getOutputAddress(), std::move(Map)); ReverseMap.emplace(OutputAddress, InputAddress); @@ -113,7 +114,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); Map.clear(); for (const BinaryBasicBlock *const BB : FF) - writeEntriesForBB(Map, *BB, FF.getAddress()); + writeEntriesForBB(Map, *BB, InputAddress, FF.getAddress()); Maps.emplace(FF.getAddress(), std::move(Map)); } @@ -125,11 +126,9 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { writeMaps(Maps, PrevAddress, OS); BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; - const uint64_t NumBBHashes = std::accumulate( - FuncHashes.begin(), FuncHashes.end(), 0ull, - [](size_t Acc, const auto &B) { return Acc + B.second.second.size(); }); - BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.size() << " function and " - << NumBBHashes << " basic block hashes\n"; + BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.getNumFunctions() + << " function and " << FuncHashes.getNumBasicBlocks() + << " basic block hashes\n"; } APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map, @@ -176,11 +175,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Only process cold fragments in cold mode, and vice versa. if (Cold != ColdPartSource.count(Address)) continue; - // NB: here we use the input address because hashes are saved early (in - // `saveMetadata`) before output addresses are assigned. + // NB: in `writeMaps` we use the input address because hashes are saved + // early in `saveMetadata` before output addresses are assigned. const uint64_t HotInputAddress = ReverseMap[Cold ? ColdPartSource[Address] : Address]; - std::pair &FuncHashPair = FuncHashes[HotInputAddress]; MapTy &Map = MapEntry.second; const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" @@ -194,10 +192,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, PrevIndex = HotIndex; } else { // Function hash - LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); - OS.write(reinterpret_cast(&FuncHashPair.first), 8); + size_t BFHash = getBFHash(HotInputAddress); + LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", BFHash)); + OS.write(reinterpret_cast(&BFHash), 8); // Number of basic blocks - size_t NumBasicBlocks = FuncHashPair.second.size(); + size_t NumBasicBlocks = getBBHashMap(HotInputAddress).getNumBasicBlocks(); LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); encodeULEB128(NumBasicBlocks, OS); } @@ -221,6 +220,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, }); } } + const BBHashMapTy &BBHashMap = getBBHashMap(HotInputAddress); size_t Index = 0; uint64_t InOffset = 0; size_t PrevBBIndex = 0; @@ -233,9 +233,9 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + const bool IsBlock = BBHashMap.isInputBlock(InOffset >> 1); + unsigned BBIndex = IsBlock ? BBHashMap.getBBIndex(InOffset >> 1) : 0; + size_t BBHash = IsBlock ? BBHashMap.getBBHash(InOffset >> 1) : 0; OS.write(reinterpret_cast(&BBHash), 8); // Basic block index in the input binary encodeULEB128(BBIndex - PrevBBIndex, OS); @@ -295,7 +295,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, HotFuncs.push_back(Address); // Function hash const size_t FuncHash = DE.getU64(&Offset, &Err); - FuncHashes[Address].first = FuncHash; + FuncHashes.addEntry(Address, FuncHash); LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); // Number of basic blocks const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); @@ -355,8 +355,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, BBIndexDelta = DE.getULEB128(&Offset, &Err); BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + getBBHashMap(HotAddress).addEntry(InputOffset >> 1, BBIndex, BBHash); } LLVM_DEBUG({ dbgs() << formatv( @@ -385,6 +384,8 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { OS << formatv(", hash: {0:x}", getBFHash(Address)); OS << "\n"; OS << "BB mappings:\n"; + const BBHashMapTy &BBHashMap = + getBBHashMap(HotAddress ? HotAddress : Address); for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit @@ -393,8 +394,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { if (IsBranch) OS << " (branch)"; else - OS << formatv(" hash: {0:x}", - getBBHash(HotAddress ? HotAddress : Address, Val)); + OS << formatv(" hash: {0:x}", BBHashMap.getBBHash(Val)); OS << "\n"; } OS << "\n"; @@ -515,27 +515,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple())) continue; // Prepare function and block hashes - FuncHashes[BF.getAddress()].first = BF.computeHash(); + FuncHashes.addEntry(BF.getAddress(), BF.computeHash()); BF.computeBlockHashes(); + BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress()); + // Set BF/BB metadata for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - -size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; -} - -size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { - return FuncHashes.at(OutputAddress).first; -} - } // namespace bolt } // namespace llvm -- GitLab From 90a7fc366ad05a7dd0465730c055af57c80d62d9 Mon Sep 17 00:00:00 2001 From: "Felix (Ting Wang)" Date: Sun, 24 Mar 2024 08:46:45 +0800 Subject: [PATCH 029/879] [PowerPC][NFC] Add base test case for small-local-dynamic-tls on AIX (#84711) --- ...aix-small-local-dynamic-tls-largeaccess.ll | 632 ++++++++++ .../aix-small-local-dynamic-tls-types.ll | 1066 +++++++++++++++++ 2 files changed, 1698 insertions(+) create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll new file mode 100644 index 000000000000..eb16bae67150 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll @@ -0,0 +1,632 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 + +; Test disassembly of object. +; RUN: llc -verify-machineinstrs -mcpu=pwr7 \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff -xcoff-traceback-table=false \ +; RUN: --code-model=large -filetype=obj -o %t.o < %s +; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck -D#NFA=2 --check-prefix=DIS %s + +@ElementIntTLSv1 = thread_local(localdynamic) global [8187 x i32] zeroinitializer, align 4 ; Within 32K +@ElementIntTLS2 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS3 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS4 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS5 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLSv2 = thread_local(localdynamic) global [9000 x i32] zeroinitializer, align 4 ; Beyond 32K + +@ElementLongTLS6 = external thread_local(localdynamic) global [60 x i64], align 8 +@ElementLongTLS2 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 ; Within 32K +@MyTLSGDVar = thread_local global [800 x i64] zeroinitializer, align 8 +@ElementLongTLS3 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS4 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS5 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS = thread_local(localdynamic) local_unnamed_addr global [7800 x i64] zeroinitializer, align 8 ; Beyond 32K + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16. +define signext i32 @test1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C5(r2) # target-flags(ppc-tlsld) @ElementIntTLSv1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv1) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [8187 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add6 = add i32 %add, %load2 + %add8 = add i32 %add6, %load3 + %add10 = add i32 %add8, %load4 + %add12 = add i32 %add10, %load5 + ret i32 %add12 +} + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16ds. +define i64 @test2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @ElementLongTLS6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 212 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 424(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C7(r2) # target-flags(ppc-tlsld) @ElementLongTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C8(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C10(r2) # target-flags(ppc-tlsld) @ElementLongTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @ElementLongTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 100 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 6800(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C12(r2) # target-flags(ppc-tlsld) @ElementLongTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 212 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C6@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 424(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C8@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C10@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 100 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C11@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 6800(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C12@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS6) + %arrayidx = getelementptr inbounds [60 x i64], ptr %tls1, i64 0, i64 53 + store i64 212, ptr %arrayidx, align 8 + %tls2 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS2) + %arrayidx1 = getelementptr inbounds [3000 x i64], ptr %tls2, i64 0, i64 150 + store i64 203, ptr %arrayidx1, align 8 + %tls3 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @MyTLSGDVar) + %arrayidx2 = getelementptr inbounds [800 x i64], ptr %tls3, i64 0, i64 55 + store i64 44, ptr %arrayidx2, align 8 + %tls4 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS3) + %arrayidx3 = getelementptr inbounds [3000 x i64], ptr %tls4, i64 0, i64 250 + store i64 6, ptr %arrayidx3, align 8 + %tls5 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS4) + %arrayidx4 = getelementptr inbounds [3000 x i64], ptr %tls5, i64 0, i64 850 + store i64 100, ptr %arrayidx4, align 8 + %tls6 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS5) + %arrayidx5 = getelementptr inbounds [3000 x i64], ptr %tls6, i64 0, i64 1050 + store i64 882, ptr %arrayidx5, align 8 + %load1 = load i64, ptr %arrayidx1, align 8 + %load2 = load i64, ptr %arrayidx3, align 8 + %load3 = load i64, ptr %arrayidx4, align 8 + %add = add i64 %load1, 882 + %add9 = add i64 %add, %load2 + %add11 = add i64 %add9, %load3 + ret i64 %add11 +} + +; Example of one access using the regular local-dynamic access from the TOC. +define signext i32 @test3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C13(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C13@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv2) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [9000 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add9 = add i32 %add, %load2 + %add11 = add i32 %add9, %load3 + %add13 = add i32 %add11, %load4 + %add15 = add i32 %add13, %load5 + ret i32 %add15 +} + +; DIS: file format aix5coff64-rs6000 +; DIS: Disassembly of section .text: +; DIS: 0000000000000000 (idx: [[#NFA+9]]) .test1: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 8(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000090 (idx: [[#NFA+11]]) .test2: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 212 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mr 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 48(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 424(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 203 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 56(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 1200(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 4, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 64(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 4, 72(4) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+3]]) .__tls_get_addr[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 44 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 440(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 80(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 2000(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 100 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 88(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 6800(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 882 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 96(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 8400(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 1191 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000140 (idx: [[#NFA+13]]) .test3: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 104(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: Disassembly of section .data: + +; DIS: 00000000000001d0 (idx: 17) test1[DS]: +; DIS-NEXT: 1d0: 00 00 00 00 +; DIS-NEXT: 00000000000001d0: R_POS (idx: [[#NFA+9]]) .test1 +; DIS-NEXT: 1d4: 00 00 00 00 +; DIS-NEXT: 1d8: 00 00 00 00 +; DIS-NEXT: 00000000000001d8: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1dc: 00 00 02 18 + +; DIS: 00000000000001e8 (idx: 19) test2[DS]: +; DIS-NEXT: 1e8: 00 00 00 00 +; DIS-NEXT: 00000000000001e8: R_POS (idx: [[#NFA+11]]) .test2 +; DIS-NEXT: 1ec: 00 00 00 90 +; DIS-NEXT: 1f0: 00 00 00 00 +; DIS-NEXT: 00000000000001f0: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1f4: 00 00 02 18 + +; DIS: 0000000000000200 (idx: 21) test3[DS]: +; DIS-NEXT: 200: 00 00 00 00 +; DIS-NEXT: 0000000000000200: R_POS (idx: [[#NFA+13]]) .test3 +; DIS-NEXT: 204: 00 00 01 40 +; DIS-NEXT: 208: 00 00 00 00 +; DIS-NEXT: 0000000000000208: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 20c: 00 00 02 18 + +; DIS: 0000000000000218 (idx: 25) _$TLSML[TC]: +; DIS-NEXT: 218: 00 00 00 00 +; DIS-NEXT: 0000000000000218: R_TLSML (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: 21c: 00 00 00 00 + +; DIS: 0000000000000220 (idx: 27) ElementIntTLSv1[TE]: +; DIS-NEXT: 220: 00 00 00 00 +; DIS-NEXT: 0000000000000220: R_TLS_LD (idx: [[#NFA+51]]) ElementIntTLSv1[TL] +; DIS-NEXT: 224: 00 00 00 00 + +; DIS: 0000000000000228 (idx: 29) ElementIntTLS3[TE]: +; DIS-NEXT: 228: 00 00 00 00 +; DIS-NEXT: 0000000000000228: R_TLS_LD (idx: [[#NFA+55]]) ElementIntTLS3[TL] +; DIS-NEXT: 22c: 00 00 be 6c + +; DIS: 0000000000000230 (idx: 31) ElementIntTLS4[TE]: +; DIS-NEXT: 230: 00 00 00 00 +; DIS-NEXT: 0000000000000230: R_TLS_LD (idx: [[#NFA+57]]) ElementIntTLS4[TL] +; DIS-NEXT: 234: 00 00 fc ec + +; DIS: 0000000000000238 (idx: 33) ElementIntTLS5[TE]: +; DIS-NEXT: 238: 00 00 00 00 +; DIS-NEXT: 0000000000000238: R_TLS_LD (idx: [[#NFA+59]]) ElementIntTLS5[TL] +; DIS-NEXT: 23c: 00 01 3b 6c + +; DIS: 0000000000000240 (idx: 35) ElementIntTLS2[TE]: +; DIS-NEXT: 240: 00 00 00 00 +; DIS-NEXT: 0000000000000240: R_TLS_LD (idx: [[#NFA+53]]) ElementIntTLS2[TL] +; DIS-NEXT: 244: 00 00 7f ec + +; DIS: 0000000000000248 (idx: 37) ElementLongTLS6[TE]: +; DIS-NEXT: 248: 00 00 00 00 +; DIS-NEXT: 0000000000000248: R_TLS_LD (idx: [[#NFA+5]]) ElementLongTLS6[UL] +; DIS-NEXT: 24c: 00 00 00 00 + +; DIS: 0000000000000250 (idx: 39) ElementLongTLS2[TE]: +; DIS-NEXT: 250: 00 00 00 00 +; DIS-NEXT: 0000000000000250: R_TLS_LD (idx: [[#NFA+63]]) ElementLongTLS2[TL] +; DIS-NEXT: 254: 00 02 06 90 + +; DIS: 0000000000000258 (idx: 41) .MyTLSGDVar[TE]: +; DIS-NEXT: 258: 00 00 00 00 +; DIS-NEXT: 0000000000000258: R_TLSM (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 25c: 00 00 00 00 + +; DIS: 0000000000000260 (idx: 43) MyTLSGDVar[TE]: +; DIS-NEXT: 260: 00 00 00 00 +; DIS-NEXT: 0000000000000260: R_TLS (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 264: 00 02 64 50 + +; DIS: 0000000000000268 (idx: 45) ElementLongTLS3[TE]: +; DIS-NEXT: 268: 00 00 00 00 +; DIS-NEXT: 0000000000000268: R_TLS_LD (idx: [[#NFA+67]]) ElementLongTLS3[TL] +; DIS-NEXT: 26c: 00 02 7d 50 + +; DIS: 0000000000000270 (idx: 47) ElementLongTLS4[TE]: +; DIS-NEXT: 270: 00 00 00 00 +; DIS-NEXT: 0000000000000270: R_TLS_LD (idx: [[#NFA+69]]) ElementLongTLS4[TL] +; DIS-NEXT: 274: 00 02 db 10 + +; DIS: 0000000000000278 (idx: 49) ElementLongTLS5[TE]: +; DIS-NEXT: 278: 00 00 00 00 +; DIS-NEXT: 0000000000000278: R_TLS_LD (idx: [[#NFA+71]]) ElementLongTLS5[TL] +; DIS-NEXT: 27c: 00 03 38 d0 + +; DIS: 0000000000000280 (idx: 51) ElementIntTLSv2[TE]: +; DIS-NEXT: 280: 00 00 00 00 +; DIS-NEXT: 0000000000000280: R_TLS_LD (idx: [[#NFA+61]]) ElementIntTLSv2[TL] +; DIS-NEXT: 284: 00 01 79 ec + +; DIS: Disassembly of section .tdata: +; DIS: 0000000000000000 (idx: [[#NFA+51]]) ElementIntTLSv1[TL]: +; DIS: 0000000000007fec (idx: [[#NFA+53]]) ElementIntTLS2[TL]: +; DIS: 000000000000be6c (idx: [[#NFA+55]]) ElementIntTLS3[TL]: +; DIS: 000000000000fcec (idx: [[#NFA+57]]) ElementIntTLS4[TL]: +; DIS: 0000000000013b6c (idx: [[#NFA+59]]) ElementIntTLS5[TL]: +; DIS: 00000000000179ec (idx: [[#NFA+61]]) ElementIntTLSv2[TL]: +; DIS: 0000000000020690 (idx: [[#NFA+63]]) ElementLongTLS2[TL]: +; DIS: 0000000000026450 (idx: [[#NFA+65]]) MyTLSGDVar[TL]: +; DIS: 0000000000027d50 (idx: [[#NFA+67]]) ElementLongTLS3[TL]: +; DIS: 000000000002db10 (idx: [[#NFA+69]]) ElementLongTLS4[TL]: +; DIS: 00000000000338d0 (idx: [[#NFA+71]]) ElementLongTLS5[TL]: +; DIS: 0000000000039690 (idx: [[#NFA+73]]) ElementLongTLS[TL]: diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll new file mode 100644 index 000000000000..d996d86a23d8 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll @@ -0,0 +1,1066 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0 + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 +@tlv_int_init = local_unnamed_addr global i32 87, align 4 + +@tlv_char = thread_local(localdynamic) global i8 1, align 1 +@tlv_short = thread_local(localdynamic) global i8 1, align 2 +@tlv_int = thread_local(localdynamic) global i32 1, align 4 +@internal_tlv_int = internal thread_local(localdynamic) global i32 1, align 4 +@tlv_long = thread_local(localdynamic) global i64 1, align 8 +@internal_tlv_long = internal thread_local(localdynamic) global i64 1, align 8 +@tlv_float = thread_local(localdynamic) global float 1.000000e+00, align 4 +@internal_tlv_double = internal thread_local(localdynamic) global double 1.000000e+00, align 8 + +%struct.anon = type { i32 } +@ThreadLocalStruct = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1 +@a = thread_local(localdynamic) global [87 x i32] zeroinitializer, align 4 + +define nonnull ptr @AddrTest1() local_unnamed_addr { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C0@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a) + %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 3 + ret ptr %arrayidx +} + +define signext i32 @testUnaligned() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C2@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C2@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStruct) + %x = getelementptr inbounds %struct.anon, ptr %tlv_addr, i32 0, i32 0 + %value = load i32, ptr %x, align 1 + ret i32 %value +} + +define void @testChar(i8 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C3@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C3@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @tlv_char) + store i8 %x, ptr %tlv_addr, align 1 + ret void +} + +define void @testShort(i16 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C4@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C4@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 2 ptr @llvm.threadlocal.address.p0(ptr align 2 @tlv_short) + store i16 %x, ptr %tlv_addr, align 2 + ret void +} + +define signext i32 @testInt1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C5@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_int) + %value = load i32, ptr %tlv_addr, align 4 + ret i32 %value +} + +define signext i32 @testInt2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C6@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C7@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C6@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @internal_tlv_int) + %tlv_val = load i32, ptr %tlv_addr, align 4 + %global_val = load i32, ptr @tlv_int_init, align 4 + %sum = add nsw i32 %global_val, %tlv_val + ret i32 %sum +} + +define signext i64 @testLong1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 4 + ret i64 %value +} + +define void @testLong2(i64 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %add = add nsw i64 %value, 9 + store i64 %add, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testLong3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %conv = trunc i64 %value to i32 + ret i32 %conv +} + +define void @testFloat1(float noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C11(r2) # %const.1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C12(r2) # %const.0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C11@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C12@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %inc = fadd float %value, 1.000000e+00 + %add = fadd float %inc, 8.000000e+00 + store float %add, ptr %tlv_addr, align 4 + ret void +} + +define i32 @testFloat2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %conv = fptosi float %value to i32 + ret i32 %conv +} + +define void @testDouble1(double noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + store double %x, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testDouble2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r1, 52 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 52(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 96(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r1, 68 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 68(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 80 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + %value = load double, ptr %tlv_addr, align 8 + %conv = fptosi double %value to i32 + ret i32 %conv +} -- GitLab From a118769ec8dc94b8332fe50c9240fecc8065a417 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 23 Mar 2024 20:56:45 -0400 Subject: [PATCH 030/879] [gn] port 3bc71c2abf --- llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn index efb324713cfe..d533e79e6374 100644 --- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn @@ -65,6 +65,7 @@ write_cmake_config("lit_common_configured") { "SANITIZER_USE_STATIC_CXX_ABI_PYBOOL=False", "SANITIZER_USE_STATIC_LLVM_UNWINDER_PYBOOL=False", "COMPILER_RT_HAS_AARCH64_SME_PYBOOL=False", + "COMPILER_RT_DARWIN_LINKER_VERSION=", "COMPILER_RT_HAS_LLD_PYBOOL=True", "COMPILER_RT_HAS_GWP_ASAN_PYBOOL=False", "HAVE_RPC_XDR_H=0", -- GitLab From 8e698a1d8e6984e41d2fdf159393d01951d87c21 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Sun, 24 Mar 2024 00:56:59 +0000 Subject: [PATCH 031/879] [gn build] Port b68e2eba0bc8 --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index 3c8b80526773..ac111f902513 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -287,6 +287,7 @@ if (current_toolchain == default_toolchain) { "__algorithm/shift_right.h", "__algorithm/shuffle.h", "__algorithm/sift_down.h", + "__algorithm/simd_utils.h", "__algorithm/sort.h", "__algorithm/sort_heap.h", "__algorithm/stable_partition.h", -- GitLab From d7d2f7ca62400ed4a3f8f89062d2aeec61bd29d4 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 19:11:49 -0700 Subject: [PATCH 032/879] [BOLT] Emit intra-function control flow in YAMLBAT Attach branch counters to YAML profile, covering intra-function control flow. Depends on: https://github.com/llvm/llvm-project/pull/86353 Test Plan: Updated bolt/test/X86/bolt-address-translation-yaml.test Reviewers: rafaelauler, dcci, ayermolo, maksfb Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76911 --- .../bolt/Profile/BoltAddressTranslation.h | 5 ++ bolt/lib/Profile/DataAggregator.cpp | 46 +++++++++++++++++++ .../X86/bolt-address-translation-yaml.test | 14 +++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index f8c35f8066f7..51fdadce8085 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -257,6 +257,11 @@ public: std::as_const(*this).getBBHashMap(FuncOutputAddress)); } + /// Returns the number of basic blocks in a function. + size_t getNumBasicBlocks(uint64_t OutputAddress) const { + return NumBasicBlocksMap.at(OutputAddress); + } + private: FuncHashesTy FuncHashes; }; diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 37c637a44a0e..b0f1cccc3256 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -2310,6 +2310,52 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC, BP.Functions.emplace_back( YAMLProfileWriter::convert(Function, /*UseDFS=*/false)); } + + for (const auto &KV : NamesToBranches) { + const StringRef FuncName = KV.first; + const FuncBranchData &Branches = KV.second; + yaml::bolt::BinaryFunctionProfile YamlBF; + BinaryData *BD = BC.getBinaryDataByName(FuncName); + assert(BD); + uint64_t FuncAddress = BD->getAddress(); + if (!BAT->isBATFunction(FuncAddress)) + continue; + // Filter out cold fragments + if (!BD->getSectionName().equals(BC.getMainCodeSectionName())) + continue; + BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress); + assert(BF); + YamlBF.Name = FuncName.str(); + YamlBF.Id = BF->getFunctionNumber(); + YamlBF.Hash = BAT->getBFHash(FuncAddress); + YamlBF.ExecCount = BF->getKnownExecutionCount(); + YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress); + const BoltAddressTranslation::BBHashMapTy &BlockMap = + BAT->getBBHashMap(FuncAddress); + + auto addSuccProfile = [&](yaml::bolt::BinaryBasicBlockProfile &YamlBB, + uint64_t SuccOffset, unsigned SuccDataIdx) { + const llvm::bolt::BranchInfo &BI = Branches.Data.at(SuccDataIdx); + yaml::bolt::SuccessorInfo SI; + SI.Index = BlockMap.getBBIndex(SuccOffset); + SI.Count = BI.Branches; + SI.Mispreds = BI.Mispreds; + YamlBB.Successors.emplace_back(SI); + }; + + for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) { + yaml::bolt::BinaryBasicBlockProfile YamlBB; + if (!BlockMap.isInputBlock(FromOffset)) + continue; + YamlBB.Index = BlockMap.getBBIndex(FromOffset); + YamlBB.Hash = BlockMap.getBBHash(FromOffset); + for (const auto &[SuccOffset, SuccDataIdx] : SuccKV) + addSuccProfile(YamlBB, SuccOffset, SuccDataIdx); + if (YamlBB.ExecCount || !YamlBB.Successors.empty()) + YamlBF.Blocks.emplace_back(YamlBB); + } + BP.Functions.emplace_back(YamlBF); + } } // Write the profile. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index ee54a90a9f2a..be3bcb2a0c6d 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -25,6 +25,7 @@ READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries YAML-BAT-CHECK: functions: +# Function not covered by BAT - has insns in basic block YAML-BAT-CHECK: - name: main YAML-BAT-CHECK-NEXT: fid: 2 YAML-BAT-CHECK-NEXT: hash: 0x9895746D48B2C876 @@ -35,6 +36,17 @@ YAML-BAT-CHECK-NEXT: - bid: 0 YAML-BAT-CHECK-NEXT: insns: 26 YAML-BAT-CHECK-NEXT: hash: 0xA900AE79CFD40000 YAML-BAT-CHECK-NEXT: succ: [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ] +# Function covered by BAT - doesn't have insns in basic block +YAML-BAT-CHECK: - name: usqrt +YAML-BAT-CHECK-NEXT: fid: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0x99E67ED32A203023 +YAML-BAT-CHECK-NEXT: exec: 21 +YAML-BAT-CHECK-NEXT: nblocks: 5 +YAML-BAT-CHECK-NEXT: blocks: +YAML-BAT-CHECK: - bid: 1 +YAML-BAT-CHECK-NEXT: insns: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0xD70DC695320E0010 +YAML-BAT-CHECK-NEXT: succ: {{.*}} { bid: 2, cnt: [[#]] } CHECK-BOLT-YAML: pre-processing profile using YAML profile reader -CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile +CHECK-BOLT-YAML-NEXT: 5 out of 16 functions in the binary (31.2%) have non-empty execution profile -- GitLab From 7c9b5228da94a44f5e3948814d896de537d162bb Mon Sep 17 00:00:00 2001 From: Owen Anderson Date: Sat, 23 Mar 2024 21:49:29 -0500 Subject: [PATCH 033/879] Only check assertions that were meant to apply to the normal case of non-splat vector SREM expansion when we aren't hitting the special case. (#86238) Fixes https://github.com/llvm/llvm-project/issues/84830 Introduced in https://github.com/llvm/llvm-project/pull/82706 --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++----- llvm/test/CodeGen/AArch64/srem-vec-crash.ll | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/srem-vec-crash.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index da29b1d5b312..8be03b66e155 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -6916,6 +6916,11 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // Q = floor((2 * A) / (2^K)) APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K)); + assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && + "We are expecting that A is always less than all-ones for SVT"); + assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && + "We are expecting that K is always less than all-ones for ShSVT"); + // If D was a power of two, apply the alternate constant derivation. if (D0.isOne()) { // A = 2^(W-1) @@ -6924,11 +6929,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, Q = APInt::getAllOnes(W - K).zext(W); } - assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && - "We are expecting that A is always less than all-ones for SVT"); - assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && - "We are expecting that K is always less than all-ones for ShSVT"); - // If the divisor is 1 the result can be constant-folded. Likewise, we // don't care about INT_MIN lanes, those can be set to undef if appropriate. if (D.isOne()) { diff --git a/llvm/test/CodeGen/AArch64/srem-vec-crash.ll b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll new file mode 100644 index 000000000000..0fce8de30d4d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s + +define i32 @pr84830(i1 %arg) { +; CHECK-LABEL: pr84830: +; CHECK: // %bb.0: // %bb +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +bb: + %new0 = srem i1 %arg, true + %last = zext i1 %new0 to i32 + %i = icmp ne i32 %last, 0 + %i1 = select i1 %i, i32 0, i32 1 + ret i32 %i1 +} -- GitLab From 74799f424063a2d751e0f9ea698db1f4efd0d8b2 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 23 Mar 2024 19:50:15 -0700 Subject: [PATCH 034/879] [memprof] Add call stack IDs to IndexedAllocationInfo (#85888) The indexed MemProf file has a huge amount of redundancy. In a large internal application, 82% of call stacks, stored in IndexedAllocationInfo::CallStack, are duplicates. We should work toward deduplicating call stacks by referring to them with unique IDs with actual call stacks stored in a separate data structure, much like we refer to memprof::Frame with memprof::FrameId. At the same time, we need to facilitate a graceful transition from the current version of the MemProf format to the next. We should be able to read (but not write) the current version of the MemProf file even after we move onto the next one. With those goals in mind, I propose to have an integer ID next to CallStack in IndexedAllocationInfo to refer to a call stack in a succinct manner. We'll gradually increase the areas of the compiler where IDs and call stacks have one-to-one correspondence and eventually remove the existing CallStack field. This patch adds call stack ID, named CSId, to IndexedAllocationInfo and teaches the raw profile reader to compute unique call stack IDs and store them in the new field. It does not introduce any user of the call stack IDs yet, except in verifyFunctionProfileData. --- llvm/include/llvm/ProfileData/MemProf.h | 23 ++++++++++++++++-- llvm/lib/ProfileData/MemProf.cpp | 25 ++++++++++++++++++++ llvm/lib/ProfileData/RawMemProfReader.cpp | 6 ++++- llvm/unittests/ProfileData/InstrProfTest.cpp | 3 ++- llvm/unittests/ProfileData/MemProfTest.cpp | 7 ++++-- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 37c19094bc2a..75ea0e49d453 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -1,6 +1,7 @@ #ifndef LLVM_PROFILEDATA_MEMPROF_H_ #define LLVM_PROFILEDATA_MEMPROF_H_ +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/GlobalValue.h" @@ -252,18 +253,26 @@ struct Frame { } }; +// A type representing the index into the table of call stacks. +using CallStackId = uint64_t; + // Holds allocation information in a space efficient format where frames are // represented using unique identifiers. struct IndexedAllocationInfo { // The dynamic calling context for the allocation in bottom-up (leaf-to-root) // order. Frame contents are stored out-of-line. + // TODO: Remove once we fully transition to CSId. llvm::SmallVector CallStack; + // Conceptually the same as above. We are going to keep both CallStack and + // CallStackId while we are transitioning from CallStack to CallStackId. + CallStackId CSId = 0; // The statistics obtained from the runtime for the allocation. PortableMemInfoBlock Info; IndexedAllocationInfo() = default; - IndexedAllocationInfo(ArrayRef CS, const MemInfoBlock &MB) - : CallStack(CS.begin(), CS.end()), Info(MB) {} + IndexedAllocationInfo(ArrayRef CS, CallStackId CSId, + const MemInfoBlock &MB) + : CallStack(CS.begin(), CS.end()), CSId(CSId), Info(MB) {} // Returns the size in bytes when this allocation info struct is serialized. size_t serializedSize() const { @@ -622,6 +631,16 @@ public: return Frame::deserialize(D); } }; + +// Compute a CallStackId for a given call stack. +CallStackId hashCallStack(ArrayRef CS); + +// Verify that each CallStackId is computed with hashCallStack. This function +// is intended to help transition from CallStack to CSId in +// IndexedAllocationInfo. +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData); } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 0461f0e9f840..bffa4ed19be8 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -3,8 +3,10 @@ #include "llvm/IR/Function.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/BLAKE3.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/HashBuilder.h" namespace llvm { namespace memprof { @@ -117,5 +119,28 @@ Expected readMemProfSchema(const unsigned char *&Buffer) { return Result; } +CallStackId hashCallStack(ArrayRef CS) { + llvm::HashBuilder, llvm::endianness::little> + HashBuilder; + for (FrameId F : CS) + HashBuilder.add(F); + llvm::BLAKE3Result<8> Hash = HashBuilder.final(); + CallStackId CSId; + std::memcpy(&CSId, Hash.data(), sizeof(Hash)); + return CSId; +} + +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData) { + for (const auto &[GUID, Record] : FunctionProfileData) { + (void)GUID; + for (const auto &AS : Record.AllocSites) { + assert(AS.CSId == hashCallStack(AS.CallStack)); + (void)AS; + } + } +} + } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp index 60c37c417aa0..5dc1ff897815 100644 --- a/llvm/lib/ProfileData/RawMemProfReader.cpp +++ b/llvm/lib/ProfileData/RawMemProfReader.cpp @@ -446,6 +446,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { Callstack.append(Frames.begin(), Frames.end()); } + CallStackId CSId = hashCallStack(Callstack); + // We attach the memprof record to each function bottom-up including the // first non-inline frame. for (size_t I = 0; /*Break out using the condition below*/; I++) { @@ -453,7 +455,7 @@ Error RawMemProfReader::mapRawProfileToRecords() { auto Result = FunctionProfileData.insert({F.Function, IndexedMemProfRecord()}); IndexedMemProfRecord &Record = Result.first->second; - Record.AllocSites.emplace_back(Callstack, Entry.second); + Record.AllocSites.emplace_back(Callstack, CSId, Entry.second); if (!F.IsInlineFrame) break; @@ -471,6 +473,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { } } + verifyFunctionProfileData(FunctionProfileData); + return Error::success(); } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index cd4552a039b3..c9323420bda7 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -366,7 +366,8 @@ IndexedMemProfRecord makeRecord( const MemInfoBlock &Block = MemInfoBlock()) { llvm::memprof::IndexedMemProfRecord MR; for (const auto &Frames : AllocFrames) - MR.AllocSites.emplace_back(Frames, Block); + MR.AllocSites.emplace_back(Frames, llvm::memprof::hashCallStack(Frames), + Block); for (const auto &Frames : CallSiteFrames) MR.CallSites.push_back(Frames); return MR; diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index f5e4a4aff2ed..1cca44e9b037 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -280,7 +280,8 @@ TEST(MemProf, RecordSerializationRoundTrip) { IndexedMemProfRecord Record; for (const auto &ACS : AllocCallStacks) { // Use the same info block for both allocation sites. - Record.AllocSites.emplace_back(ACS, Info); + Record.AllocSites.emplace_back(ACS, llvm::memprof::hashCallStack(ACS), + Info); } Record.CallSites.assign(CallSites); @@ -376,7 +377,9 @@ TEST(MemProf, BaseMemProfReader) { Block.AllocCount = 1U, Block.TotalAccessDensity = 4, Block.TotalLifetime = 200001; std::array CallStack{F1.hash(), F2.hash()}; - FakeRecord.AllocSites.emplace_back(/*CS=*/CallStack, /*MB=*/Block); + FakeRecord.AllocSites.emplace_back( + /*CS=*/CallStack, /*CSId=*/llvm::memprof::hashCallStack(CallStack), + /*MB=*/Block); ProfData.insert({F1.hash(), FakeRecord}); MemProfReader Reader(FrameIdMap, ProfData); -- GitLab From a45e58af1b381cf3c0374332386b8291ec5310f4 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Sun, 24 Mar 2024 12:48:19 +0900 Subject: [PATCH 035/879] [mlir][bufferization] Add `BufferViewFlowOpInterface` (#78718) This commit adds the `BufferViewFlowOpInterface` to the bufferization dialect. This interface can be implemented by ops that operate on buffers to indicate that a buffer op result and/or region entry block argument may be the same buffer as a buffer operand (or a view thereof). This interface is queried by the `BufferViewFlowAnalysis`. The new interface has two interface methods: * `populateDependencies`: Implementations use the provided callback to declare dependencies between operands and op results/region entry block arguments. E.g., for `%r = arith.select %c, %m1, %m2 : memref<5xf32>`, the interface implementation should declare two dependencies: %m1 -> %r and %m2 -> %r. * `mayBeTerminalBuffer`: An SSA value is a terminal buffer if the buffer view flow analysis stops at the specified value. E.g., because the value is a newly allocated buffer or because no further information is available about the origin of the buffer. Ops that implement the `RegionBranchOpInterface` or `BranchOpInterface` do not have to implement the `BufferViewFlowOpInterface`. The buffer dependencies can be inferred from those two interfaces. This commit makes the `BufferViewFlowAnalysis` more accurate. For unknown ops, it conservatively used to declare all combinations of operands and op results/region entry block arguments as dependencies (false positives). This is no longer the case. While the analysis is still a "maybe" analysis with false positives (e.g., when analyzing ops such as `arith.select` or `scf.if` where the taken branch is not known at compile time), results and region entry block arguments of unknown ops are now marked as terminal buffers. This commit addresses a TODO in `BufferViewFlowAnalysis.cpp`: ``` // TODO: We should have an op interface instead of a hard-coded list of // interfaces/ops. ``` It is no longer needed to hard-code ops. --- .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ .../IR/BufferViewFlowOpInterface.h | 27 +++++++ .../IR/BufferViewFlowOpInterface.td | 73 +++++++++++++++++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.h | 6 ++ .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ mlir/include/mlir/InitAllDialects.h | 4 + .../BufferViewFlowOpInterfaceImpl.cpp | 44 +++++++++++ .../Dialect/Arith/Transforms/CMakeLists.txt | 1 + .../IR/BufferViewFlowOpInterface.cpp | 18 +++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.cpp | 72 ++++++++++++++---- .../BufferViewFlowOpInterfaceImpl.cpp | 48 ++++++++++++ .../Dialect/MemRef/Transforms/CMakeLists.txt | 2 + .../llvm-project-overlay/mlir/BUILD.bazel | 40 ++++++++++ 15 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td create mode 100644 mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp create mode 100644 mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp create mode 100644 mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..a2b3a9bb655b --- /dev/null +++ b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace arith { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace arith +} // namespace mlir + +#endif // MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h new file mode 100644 index 000000000000..84e67fe72b62 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h @@ -0,0 +1,27 @@ +//===- BufferViewFlowOpInterface.h - Buffer View Flow Analysis --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ +#define MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ + +#include "mlir/IR/OpDefinition.h" +#include "mlir/Support/LLVM.h" + +namespace mlir { +class ValueRange; + +namespace bufferization { + +using RegisterDependenciesFn = std::function; + +} // namespace bufferization +} // namespace mlir + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc" + +#endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td new file mode 100644 index 000000000000..58885d742266 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td @@ -0,0 +1,73 @@ +//===-- BufferViewFlowOpInterface.td - Buffer View Flow ----*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef BUFFER_VIEW_FLOW_OP_INTERFACE +#define BUFFER_VIEW_FLOW_OP_INTERFACE + +include "mlir/IR/OpBase.td" + +def BufferViewFlowOpInterface : + OpInterface<"BufferViewFlowOpInterface"> { + let description = [{ + An op interface for the buffer view flow analysis. This interface describes + buffer dependencies between operands and op results/region entry block + arguments. + }]; + let cppNamespace = "::mlir::bufferization"; + let methods = [ + InterfaceMethod< + /*desc=*/[{ + Populate buffer dependencies between operands and op results/region + entry block arguments. + + Implementations should register dependencies between an operand ("X") + and an op result/region entry block argument ("Y") if Y may depend + on X. Y depends on X if Y and X are the same buffer or if Y is a + subview of X. + + Example: + ``` + %r = arith.select %c, %m1, %m2 : memref<5xf32> + ``` + In the above example, %0 may depend on %m1 or %m2 and a correct + interface implementation should call: + - "registerDependenciesFn(%m1, %r)". + - "registerDependenciesFn(%m2, %r)" + }], + /*retType=*/"void", + /*methodName=*/"populateDependencies", + /*args=*/(ins + "::mlir::bufferization::RegisterDependenciesFn" + :$registerDependenciesFn) + >, + InterfaceMethod< + /*desc=*/[{ + Return "true" if the given value may be a terminal buffer. A buffer + value is "terminal" if it cannot be traced back any further in the + buffer view flow analysis. + + Examples: A buffer could be terminal because: + - it is a newly allocated buffer (e.g., "memref.alloc"), + - or: because there is not enough compile-time information available + to make a definite decision (e.g., "memref.realloc" may reallocate + but we do not know for sure; another example are call ops where we + would have to analyze the body of the callee). + + Implementations can assume that the given SSA value is an OpResult of + this operation or a region entry block argument of this operation. + }], + /*retType=*/"bool", + /*methodName=*/"mayBeTerminalBuffer", + /*args=*/(ins "Value":$value), + /*methodBody=*/"", + /*defaultImplementation=*/"return false;" + >, + ]; +} + +#endif // BUFFER_VIEW_FLOW_OP_INTERFACE diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt index 31a553f9a32f..13a5bc370a4f 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt @@ -3,6 +3,7 @@ add_mlir_doc(BufferizationOps BufferizationOps Dialects/ -gen-dialect-doc) add_mlir_interface(AllocationOpInterface) add_mlir_interface(BufferDeallocationOpInterface) add_mlir_interface(BufferizableOpInterface) +add_mlir_interface(BufferViewFlowOpInterface) set(LLVM_TARGET_DEFINITIONS BufferizationEnums.td) mlir_tablegen(BufferizationEnums.h.inc -gen-enum-decls) diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 24825db69f90..9e43265c5dfe 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -63,6 +63,9 @@ public: /// results have to be changed. void rename(Value from, Value to); + /// Returns "true" if the given value may be a terminal. + bool mayBeTerminalBuffer(Value value) const; + private: /// This function constructs a mapping from values to its immediate /// dependencies. @@ -70,6 +73,9 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + + /// A set of all SSA values that may be terminal buffers. + DenseSet terminals; }; } // namespace mlir diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..714518a21e97 --- /dev/null +++ b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace memref { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace memref +} // namespace mlir + +#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 9bbf12d13254..c558dc53cc7f 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -21,6 +21,7 @@ #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.h" +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" #include "mlir/Dialect/ArmSME/IR/ArmSME.h" @@ -52,6 +53,7 @@ #include "mlir/Dialect/MemRef/IR/MemRefMemorySlot.h" #include "mlir/Dialect/MemRef/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.h" +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/RuntimeOpVerification.h" #include "mlir/Dialect/Mesh/IR/MeshDialect.h" #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" @@ -148,6 +150,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { affine::registerValueBoundsOpInterfaceExternalModels(registry); arith::registerBufferDeallocationOpInterfaceExternalModels(registry); arith::registerBufferizableOpInterfaceExternalModels(registry); + arith::registerBufferViewFlowOpInterfaceExternalModels(registry); arith::registerValueBoundsOpInterfaceExternalModels(registry); bufferization::func_ext::registerBufferizableOpInterfaceExternalModels( registry); @@ -157,6 +160,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { gpu::registerBufferDeallocationOpInterfaceExternalModels(registry); linalg::registerAllDialectInterfaceImplementations(registry); memref::registerAllocationOpInterfaceExternalModels(registry); + memref::registerBufferViewFlowOpInterfaceExternalModels(registry); memref::registerRuntimeVerifiableOpInterfaceExternalModels(registry); memref::registerValueBoundsOpInterfaceExternalModels(registry); memref::registerMemorySlotExternalModels(registry); diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..9df9df86b64f --- /dev/null +++ b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,44 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace arith { +namespace { + +struct SelectOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto selectOp = cast(op); + + // Either one of the true/false value may be selected at runtime. + registerDependenciesFn(selectOp.getTrueValue(), selectOp.getResult()); + registerDependenciesFn(selectOp.getFalseValue(), selectOp.getResult()); + } +}; + +} // namespace +} // namespace arith +} // namespace mlir + +void arith::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) { + SelectOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt index 02240601bcd3..12659eaba1fa 100644 --- a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_mlir_dialect_library(MLIRArithTransforms BufferDeallocationOpInterfaceImpl.cpp BufferizableOpInterfaceImpl.cpp Bufferize.cpp + BufferViewFlowOpInterfaceImpl.cpp EmulateUnsupportedFloats.cpp EmulateWideInt.cpp EmulateNarrowType.cpp diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp new file mode 100644 index 000000000000..ea726a4bfc3f --- /dev/null +++ b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp @@ -0,0 +1,18 @@ +//===- BufferViewFlowOpInterface.cpp - Buffer View Flow Analysis ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" + +namespace mlir { +namespace bufferization { + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc" + +} // namespace bufferization +} // namespace mlir diff --git a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt index 9895db9d93ce..63dcc1eb233e 100644 --- a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt @@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRBufferizationDialect BufferDeallocationOpInterface.cpp BufferizationOps.cpp BufferizationDialect.cpp + BufferViewFlowOpInterface.cpp UnstructuredControlFlow.cpp ADDITIONAL_HEADER_DIRS diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 88ef1b639fc5..9a36057425f3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -8,12 +8,16 @@ #include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Interfaces/CallInterfaces.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" +#include "mlir/Interfaces/FunctionInterfaces.h" #include "mlir/Interfaces/ViewLikeInterface.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" using namespace mlir; +using namespace mlir::bufferization; /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } @@ -65,18 +69,44 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) this->dependencies[value].insert(dep); }; + // Mark all buffer results and buffer region entry block arguments of the + // given op as terminals. + auto populateTerminalValues = [&](Operation *op) { + for (Value v : op->getResults()) + if (isa(v.getType())) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType())) + this->terminals.insert(v); + }; + op->walk([&](Operation *op) { - // TODO: We should have an op interface instead of a hard-coded list of - // interfaces/ops. + // Query BufferViewFlowOpInterface. If the op does not implement that + // interface, try to infer the dependencies from other interfaces that the + // op may implement. + if (auto bufferViewFlowOp = dyn_cast(op)) { + bufferViewFlowOp.populateDependencies(registerDependencies); + for (Value v : op->getResults()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + return WalkResult::advance(); + } // Add additional dependencies created by view changes to the alias list. if (auto viewInterface = dyn_cast(op)) { - dependencies[viewInterface.getViewSource()].insert( - viewInterface->getResult(0)); + registerDependencies(viewInterface.getViewSource(), + viewInterface->getResult(0)); return WalkResult::advance(); } @@ -131,16 +161,30 @@ void BufferViewFlowAnalysis::build(Operation *op) { return WalkResult::advance(); } - // Unknown op: Assume that all operands alias with all results. - for (Value operand : op->getOperands()) { - if (!isa(operand.getType())) - continue; - for (Value result : op->getResults()) { - if (!isa(result.getType())) - continue; - registerDependencies({operand}, {result}); - } + // Region terminators are handled together with RegionBranchOpInterface. + if (isa(op)) + return WalkResult::advance(); + + if (isa(op)) { + // This is an intra-function analysis. We have no information about other + // functions. Conservatively assume that each operand may alias with each + // result. Also mark the results are terminals because the function could + // return newly allocated buffers. + populateTerminalValues(op); + for (Value operand : op->getOperands()) + for (Value result : op->getResults()) + registerDependencies({operand}, {result}); + return WalkResult::advance(); } + + // We have no information about unknown ops. + populateTerminalValues(op); + return WalkResult::advance(); }); } + +bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { + assert(isa(value.getType()) && "expected memref"); + return terminals.contains(value); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..bbb269bd0016 --- /dev/null +++ b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,48 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace memref { +namespace { + +struct ReallocOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto reallocOp = cast(op); + // memref.realloc may return the source operand. + registerDependenciesFn(reallocOp.getSource(), reallocOp.getResult()); + } + + bool mayBeTerminalBuffer(Operation *op, Value value) const { + // The return value of memref.realloc is a terminal buffer because the op + // may return a newly allocated buffer. + return true; + } +}; + +} // namespace +} // namespace memref +} // namespace mlir + +void memref::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, memref::MemRefDialect *dialect) { + ReallocOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt index 08b7eab726eb..f150ac7ac2d6 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt @@ -1,5 +1,6 @@ add_mlir_dialect_library(MLIRMemRefTransforms AllocationOpInterfaceImpl.cpp + BufferViewFlowOpInterfaceImpl.cpp ComposeSubView.cpp ExpandOps.cpp ExpandRealloc.cpp @@ -27,6 +28,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms MLIRArithDialect MLIRArithTransforms MLIRBufferizationDialect + MLIRBufferizationTransforms MLIRDialectUtils MLIRFuncDialect MLIRGPUDialect diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 5b6e4678a05e..88b46bdb326c 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -10828,6 +10828,36 @@ gentbl_cc_library( ], ) +td_library( + name = "BufferViewFlowOpInterfaceTdFiles", + srcs = [ + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + ], + includes = ["include"], + deps = [ + ":OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "BufferViewFlowOpInterfaceIncGen", + tbl_outs = [ + ( + ["-gen-op-interface-decls"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc", + ), + ( + ["-gen-op-interface-defs"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + deps = [ + ":BufferViewFlowOpInterfaceTdFiles", + ], +) + td_library( name = "SubsetOpInterfaceTdFiles", srcs = [ @@ -12977,6 +13007,8 @@ cc_library( ":ArithTransforms", ":ArithUtils", ":BufferizationDialect", + ":BufferizationInterfaces", + ":BufferizationTransforms", ":ControlFlowDialect", ":DialectUtils", ":FuncDialect", @@ -13369,6 +13401,7 @@ td_library( includes = ["include"], deps = [ ":AllocationOpInterfaceTdFiles", + ":BufferViewFlowOpInterfaceTdFiles", ":BufferizableOpInterfaceTdFiles", ":CopyOpInterfaceTdFiles", ":DestinationStyleOpInterfaceTdFiles", @@ -13515,11 +13548,13 @@ cc_library( ], hdrs = [ "include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h", + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h", "include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h", ], includes = ["include"], deps = [ ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationEnumsIncGen", ":IR", @@ -13532,6 +13567,7 @@ cc_library( name = "BufferizationDialect", srcs = [ "lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp", + "lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizationDialect.cpp", "lib/Dialect/Bufferization/IR/BufferizationOps.cpp", @@ -13549,10 +13585,12 @@ cc_library( ":Analysis", ":ArithDialect", ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationBaseIncGen", ":BufferizationInterfaces", ":BufferizationOpsIncGen", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":CopyOpInterface", ":DestinationStyleOpInterface", @@ -13602,9 +13640,11 @@ cc_library( ":BufferizationDialect", ":BufferizationInterfaces", ":BufferizationPassIncGen", + ":CallOpInterfaces", ":ControlFlowDialect", ":ControlFlowInterfaces", ":FuncDialect", + ":FunctionInterfaces", ":IR", ":LoopLikeInterface", ":MemRefDialect", -- GitLab From 5d7fd6a04a6748936dece9d90481b2ba4ec97e53 Mon Sep 17 00:00:00 2001 From: yingopq <115543042+yingopq@users.noreply.github.com> Date: Sun, 24 Mar 2024 14:35:42 +0800 Subject: [PATCH 036/879] [Mips] Restore wrong deletion of instruction 'and' in unsigned min/max processing. (#85902) Fix #61881 --- llvm/lib/Target/Mips/MipsExpandPseudo.cpp | 9 ++++ llvm/test/CodeGen/Mips/atomic-min-max.ll | 56 +++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp index bded59439a73..c30129743a96 100644 --- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp +++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp @@ -500,6 +500,15 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( .addReg(Incr, RegState::Kill) .addImm(ShiftImm); } + } else { + // and OldVal, OldVal, Mask + // and Incr, Incr, Mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal) + .addReg(OldVal) + .addReg(Mask); + BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr) + .addReg(Incr) + .addReg(Mask); } } // unsigned: sltu Scratch4, oldVal, Incr diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll index bc3643f3947a..a96581bdb39a 100644 --- a/llvm/test/CodeGen/Mips/atomic-min-max.ll +++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll @@ -2146,6 +2146,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB6_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -2186,6 +2188,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB6_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -2225,6 +2229,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB6_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -2263,6 +2269,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB6_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -2300,6 +2308,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB6_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -2417,6 +2427,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB6_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -2456,6 +2468,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB6_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -2655,6 +2669,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB7_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -2696,6 +2712,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB7_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -2735,6 +2753,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB7_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -2773,6 +2793,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB7_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -2810,6 +2832,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB7_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -2927,6 +2951,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB7_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -2966,6 +2992,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB7_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 @@ -4244,6 +4272,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB10_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -4284,6 +4314,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB10_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -4323,6 +4355,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB10_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -4361,6 +4395,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB10_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -4398,6 +4434,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB10_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -4515,6 +4553,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB10_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -4554,6 +4594,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB10_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -4753,6 +4795,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB11_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -4793,6 +4837,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB11_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -4832,6 +4878,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB11_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -4870,6 +4918,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB11_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -4907,6 +4957,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB11_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -5024,6 +5076,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB11_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -5063,6 +5117,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB11_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 -- GitLab From 4acd84e7ccce6a2865f60cd2adc37a335d4f35ce Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 24 Mar 2024 01:22:48 -0700 Subject: [PATCH 037/879] Revert "[compiler-rt] Also consider SIGPROF as a synchronous signal" (#86416) Reverting #85188 with follow up patches. This reverts commit 362d26366d0175f01ffb6085eb747a6e40f01147. This reverts commit c9bdeabdf4b46fbf1f6a9fcbf9cd61d460b18c08. This reverts commit 6bc6e1ace9fa8453e164fa04b5d9acd5a77e089a. This reverts commit 01fa550ff654d6724e6da54c877032baeddff14b. This reverts commit ddcbab37ac0e5743a8d39be3dd48d967f4c85504. --- compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 4 +--- compiler-rt/test/tsan/signal_errno.cpp | 6 +++--- compiler-rt/test/tsan/signal_reset.cpp | 8 ++++---- compiler-rt/test/tsan/signal_sync.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread2.cpp | 4 ++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 810ce69663d0..8ffc703b05ea 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -126,7 +126,6 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -const int SIGPROF = 27; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; @@ -2180,8 +2179,7 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { return; } // Don't mess with synchronous signals. - const bool sync = is_sync_signal(sctx, sig, info) || - (sig == SIGPROF && thr->is_inited && !thr->is_dead); + const bool sync = is_sync_signal(sctx, sig, info); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, diff --git a/compiler-rt/test/tsan/signal_errno.cpp b/compiler-rt/test/tsan/signal_errno.cpp index 99d4b6d84ca4..7e1fd4b0c5a5 100644 --- a/compiler-rt/test/tsan/signal_errno.cpp +++ b/compiler-rt/test/tsan/signal_errno.cpp @@ -18,7 +18,7 @@ static void MyHandler(int, siginfo_t *s, void *c) { static void* sendsignal(void *p) { barrier_wait(&barrier); - pthread_kill(mainth, SIGALRM); + pthread_kill(mainth, SIGPROF); return 0; } @@ -37,7 +37,7 @@ int main() { mainth = pthread_self(); struct sigaction act = {}; act.sa_sigaction = &MyHandler; - sigaction(SIGALRM, &act, 0); + sigaction(SIGPROF, &act, 0); pthread_t th; pthread_create(&th, 0, sendsignal, 0); loop(); @@ -46,7 +46,7 @@ int main() { } // CHECK: WARNING: ThreadSanitizer: signal handler spoils errno -// CHECK: Signal 14 handler invoked at: +// CHECK: Signal 27 handler invoked at: // CHECK: #0 MyHandler(int, {{(__)?}}siginfo{{(_t)?}}*, void*) {{.*}}signal_errno.cpp // CHECK: main // CHECK: SUMMARY: ThreadSanitizer: signal handler spoils errno{{.*}}MyHandler diff --git a/compiler-rt/test/tsan/signal_reset.cpp b/compiler-rt/test/tsan/signal_reset.cpp index d76b7e5f3b5f..82758d882382 100644 --- a/compiler-rt/test/tsan/signal_reset.cpp +++ b/compiler-rt/test/tsan/signal_reset.cpp @@ -28,12 +28,12 @@ static void* reset(void *p) { struct sigaction act = {}; for (int i = 0; i < 1000000; i++) { act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -44,7 +44,7 @@ static void* reset(void *p) { int main() { struct sigaction act = {}; act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -53,7 +53,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_sync.cpp b/compiler-rt/test/tsan/signal_sync.cpp index 878b3f3b88b9..b529a1859f52 100644 --- a/compiler-rt/test/tsan/signal_sync.cpp +++ b/compiler-rt/test/tsan/signal_sync.cpp @@ -30,7 +30,7 @@ int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -39,7 +39,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread.cpp b/compiler-rt/test/tsan/signal_thread.cpp index 7bba8159bf38..aa91d1ddeb10 100644 --- a/compiler-rt/test/tsan/signal_thread.cpp +++ b/compiler-rt/test/tsan/signal_thread.cpp @@ -24,7 +24,7 @@ static void* thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -33,7 +33,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread2.cpp b/compiler-rt/test/tsan/signal_thread2.cpp index 5236628e13b6..9bde4f70b39d 100644 --- a/compiler-rt/test/tsan/signal_thread2.cpp +++ b/compiler-rt/test/tsan/signal_thread2.cpp @@ -40,7 +40,7 @@ static void *thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -49,7 +49,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } -- GitLab From 4d69855e9d380ecb7c1f7a64c7b37258fe36f525 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Sun, 24 Mar 2024 06:48:04 -0400 Subject: [PATCH 038/879] [flang] Silence MSVC warning about shifts (NFC) (#83737) Yes, 64-bit shifts are intended. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index eb8f5135ff12..ea1ef1f08aba 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -3883,7 +3883,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, int pos = 3 + highSignificandSize; mlir::Value index = builder.create( loc, builder.create(loc, intVal, signShift), - createIntegerConstant(1 << pos)); + createIntegerConstant(1ULL << pos)); // [e] exponent != 0 mlir::Value exponent = @@ -3895,7 +3895,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::ne, exponent, zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [m] exponent == 1..1 (max exponent) index = builder.create( @@ -3904,7 +3904,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::eq, exponent, exponentMask), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [l] low-order significand != 0 index = builder.create( @@ -3916,7 +3916,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, builder.create(loc, intVal, lowSignificandMask), zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [h] high-order significand (1 or 2 bits) index = builder.create( -- GitLab From 6c6fe4b2aea8631001b11abee62146d4aca01cee Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 24 Mar 2024 11:33:51 +0000 Subject: [PATCH 039/879] [X86] known-never-zero.ll - add 32-bit test coverage Enabled vector coverage as well: i686+SSE2 and x64_64+AVX Should improve test quality for #85722 --- llvm/test/CodeGen/X86/known-never-zero.ll | 1708 ++++++++++++++------- 1 file changed, 1161 insertions(+), 547 deletions(-) diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll index cc9862769f2b..423516bc3271 100644 --- a/llvm/test/CodeGen/X86/known-never-zero.ll +++ b/llvm/test/CodeGen/X86/known-never-zero.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X86 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64 ;; Use cttz to test if we properly prove never-zero. There is a very ;; simple transform from cttz -> cttz_zero_undef if its operand is @@ -18,41 +19,70 @@ declare i32 @llvm.fshl.i32(i32, i32, i32) declare i32 @llvm.fshr.i32(i32, i32, i32) define i32 @or_known_nonzero(i32 %x) { -; CHECK-LABEL: or_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %z = or i32 %x, 1 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @or_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: or_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl %esi, %edi -; CHECK-NEXT: je .LBB1_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB1_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB1_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB1_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl %esi, %edi +; X64-NEXT: je .LBB1_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB1_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = or i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @select_known_nonzero(i1 %c, i32 %x) { -; CHECK-LABEL: select_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: movl $122, %eax -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: movl $122, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: testb $1, %dil +; X64-NEXT: movl $122, %eax +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 122 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -60,20 +90,36 @@ define i32 @select_known_nonzero(i1 %c, i32 %x) { } define i32 @select_maybe_zero(i1 %c, i32 %x) { -; CHECK-LABEL: select_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB3_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB3_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl $1, %ecx +; X86-NEXT: xorl %eax, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB3_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB3_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: testb $1, %dil +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB3_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB3_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 0 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -81,28 +127,45 @@ define i32 @select_maybe_zero(i1 %c, i32 %x) { } define i32 @shl_known_nonzero_1s_bit_set(i32 %x) { -; CHECK-LABEL: shl_known_nonzero_1s_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $123, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_1s_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $123, %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_1s_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $123, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = shl i32 123, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -110,14 +173,23 @@ define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -125,67 +197,116 @@ define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @shl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: shl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB7_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB7_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB7_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB7_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB7_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB7_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = shl nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_known_nonzero(i32 %x) { -; CHECK-LABEL: uaddsat_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: incl %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovnel %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: incl %eax +; X86-NEXT: movl $-1, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: incl %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovnel %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 1) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: uaddsat_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovael %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB9_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB9_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-1, %eax +; X86-NEXT: cmovael %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB9_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB9_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: addl %esi, %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovael %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB9_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB9_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umax_known_nonzero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cmpl %eax, %edi -; CHECK-NEXT: cmoval %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: cmpl %edx, %eax +; X86-NEXT: cmoval %eax, %edx +; X86-NEXT: rep bsfl %edx, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cmpl %eax, %edi +; X64-NEXT: cmoval %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %yy = shl nuw i32 4, %y %z = call i32 @llvm.umax.i32(i32 %x, i32 %yy) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -193,35 +314,62 @@ define i32 @umax_known_nonzero(i32 %x, i32 %y) { } define i32 @umax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl %esi, %edi -; CHECK-NEXT: cmoval %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB11_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB11_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl %eax, %ecx +; X86-NEXT: cmoval %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB11_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB11_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl %esi, %edi +; X64-NEXT: cmoval %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB11_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB11_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umax.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: umin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovbl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovbl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovbl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.umin.i32(i32 %x, i32 %y) @@ -230,36 +378,63 @@ define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @umin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovbl %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB13_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB13_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovbl %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB13_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB13_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovbl %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB13_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB13_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovll %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovll %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovll %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smin.i32(i32 %x, i32 %y) @@ -268,36 +443,63 @@ define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovll %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB15_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB15_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovll %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB15_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB15_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovll %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB15_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB15_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.smin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovgl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovgl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovgl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smax.i32(i32 %x, i32 %y) @@ -306,35 +508,61 @@ define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $55, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovgel %edi, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cmpl $55, %eax +; X86-NEXT: movl $54, %ecx +; X86-NEXT: cmovgel %eax, %ecx +; X86-NEXT: bsfl %ecx, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $55, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovgel %edi, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %z = call i32 @llvm.smax.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB18_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB18_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB18_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB18_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB18_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB18_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shr = lshr i32 %x, %y %sub = sub i32 32, %y @@ -345,19 +573,33 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB19_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB19_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB19_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB19_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB19_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB19_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shr = lshr i32 %x, %y %sub = sub i32 32, %y %shl = shl i32 %x, %sub @@ -367,14 +609,23 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) { } define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -382,39 +633,68 @@ define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB21_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB21_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB21_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB21_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB21_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB21_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB22_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB22_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB22_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB22_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB22_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB22_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shl = shl i32 %x, %y %sub = sub i32 32, %y @@ -425,19 +705,33 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB23_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB23_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB23_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB23_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB23_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB23_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shl = shl i32 %x, %y %sub = sub i32 32, %y %shr = lshr i32 %x, %sub @@ -447,14 +741,23 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) { } define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -462,47 +765,78 @@ define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB25_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB25_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB25_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB25_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB25_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB25_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: sra_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = ashr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: sra_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -510,47 +844,78 @@ define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @sra_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sra_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB28_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB28_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB28_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB28_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB28_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB28_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: srl_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = lshr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: srl_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -558,33 +923,56 @@ define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @srl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: srl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB31_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB31_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB31_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB31_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB31_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB31_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: udiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -592,33 +980,56 @@ define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @udiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: udiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB33_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB33_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB33_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB33_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB33_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB33_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: sdiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -626,31 +1037,53 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @sdiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sdiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB35_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB35_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB35_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB35_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB35_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB35_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @add_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nuw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -658,17 +1091,30 @@ define i32 @add_known_nonzero(i32 %xx, i32 %y) { } define i32 @add_maybe_zero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: je .LBB37_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB37_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB37_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB37_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: je .LBB37_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB37_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nsw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -676,15 +1122,24 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) { } define i32 @sub_known_nonzero_neg_case(i32 %xx) { -; CHECK-LABEL: sub_known_nonzero_neg_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: negl %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_neg_case: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: negl %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_neg_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: negl %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i32 256, %xx %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -692,14 +1147,24 @@ define i32 @sub_known_nonzero_neg_case(i32 %xx) { } define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { -; CHECK-LABEL: sub_known_nonzero_ne_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: andl $-65, %edi -; CHECK-NEXT: subl %eax, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_ne_case: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: orl $64, %ecx +; X86-NEXT: andl $-65, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_ne_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: andl $-65, %edi +; X64-NEXT: subl %eax, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %y = and i32 %xx, -65 %z = sub i32 %y, %x @@ -708,18 +1173,32 @@ define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { } define i32 @sub_maybe_zero(i32 %x) { -; CHECK-LABEL: sub_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: subl %edi, %eax -; CHECK-NEXT: je .LBB40_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB40_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: je .LBB40_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB40_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: subl %edi, %eax +; X64-NEXT: je .LBB40_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB40_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 64 %z = sub i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -727,34 +1206,60 @@ define i32 @sub_maybe_zero(i32 %x) { } define i32 @sub_maybe_zero2(i32 %x) { -; CHECK-LABEL: sub_maybe_zero2: -; CHECK: # %bb.0: -; CHECK-NEXT: negl %edi -; CHECK-NEXT: je .LBB41_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB41_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero2: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: negl %eax +; X86-NEXT: je .LBB41_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB41_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero2: +; X64: # %bb.0: +; X64-NEXT: negl %edi +; X64-NEXT: je .LBB41_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB41_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB42_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB42_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB42_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB42_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB42_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB42_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -762,18 +1267,32 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB43_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB43_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB43_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB43_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB43_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB43_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -781,36 +1300,63 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @mul_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: mul_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: imull %esi, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB44_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB44_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB44_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB44_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: imull %esi, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB44_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB44_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = mul nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_known_nonzero(<2 x i16> %xx) { -; CHECK-LABEL: bitcast_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; CHECK-NEXT: pslld $23, %xmm0 -; CHECK-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: cvttps2dq %xmm0, %xmm0 -; CHECK-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; CHECK-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; X86-NEXT: pslld $23, %xmm0 +; X86-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: cvttps2dq %xmm0, %xmm0 +; X86-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] +; X86-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: bsfl %eax, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; X64-NEXT: vpslld $23, %xmm0, %xmm0 +; X64-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vcvttps2dq %xmm0, %xmm0 +; X64-NEXT: vpackusdw %xmm0, %xmm0, %xmm0 +; X64-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %x = shl nuw nsw <2 x i16> , %xx %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -818,49 +1364,83 @@ define i32 @bitcast_known_nonzero(<2 x i16> %xx) { } define i32 @bitcast_maybe_zero(<2 x i16> %x) { -; CHECK-LABEL: bitcast_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB46_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB46_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB46_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB46_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB46_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB46_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_from_float(float %x) { -; CHECK-LABEL: bitcast_from_float: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB47_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB47_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_from_float: +; X86: # %bb.0: +; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB47_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB47_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_from_float: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB47_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB47_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast float %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @zext_known_nonzero(i16 %xx) { -; CHECK-LABEL: zext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: movzwl %ax, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -868,32 +1448,54 @@ define i32 @zext_known_nonzero(i16 %xx) { } define i32 @zext_maybe_zero(i16 %x) { -; CHECK-LABEL: zext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB49_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movzwl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB49_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testw %ax, %ax +; X86-NEXT: je .LBB49_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB49_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB49_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movzwl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB49_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sext_known_nonzero(i16 %xx) { -; CHECK-LABEL: sext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cwtl -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: cwtl +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cwtl +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -901,17 +1503,29 @@ define i32 @sext_known_nonzero(i16 %xx) { } define i32 @sext_maybe_zero(i16 %x) { -; CHECK-LABEL: sext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB51_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movswl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB51_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB51_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB51_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB51_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movswl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB51_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r -- GitLab From e8d5223ce4b2214e052b5b52b2e6453ffea0fe33 Mon Sep 17 00:00:00 2001 From: David Green Date: Sun, 24 Mar 2024 12:32:47 +0000 Subject: [PATCH 040/879] [AArch64] Additional GISel test coverage. NFC --- llvm/test/CodeGen/AArch64/setcc_knownbits.ll | 95 +++++++++++++------- 1 file changed, 65 insertions(+), 30 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll index bb9546af8bb7..6a337bf02e09 100644 --- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll +++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i1 @load_bv_v4i8(i1 zeroext %a) { ; CHECK-LABEL: load_bv_v4i8: @@ -11,18 +12,31 @@ define i1 @load_bv_v4i8(i1 zeroext %a) { } define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) { -; CHECK-LABEL: logger: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldr w8, [x2] -; CHECK-NEXT: cmp w8, w0 -; CHECK-NEXT: b.ls .LBB1_2 -; CHECK-NEXT: // %bb.1: -; CHECK-NEXT: mov w0, wzr -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB1_2: // %land.rhs -; CHECK-NEXT: ldr x8, [x1] -; CHECK-NEXT: ldrb w0, [x8] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: logger: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ldr w8, [x2] +; CHECK-SD-NEXT: cmp w8, w0 +; CHECK-SD-NEXT: b.ls .LBB1_2 +; CHECK-SD-NEXT: // %bb.1: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB1_2: // %land.rhs +; CHECK-SD-NEXT: ldr x8, [x1] +; CHECK-SD-NEXT: ldrb w0, [x8] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: logger: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr w8, [x2] +; CHECK-GI-NEXT: cmp w8, w0 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.hi .LBB1_2 +; CHECK-GI-NEXT: // %bb.1: // %land.rhs +; CHECK-GI-NEXT: ldr x8, [x1] +; CHECK-GI-NEXT: ldrb w8, [x8] +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: .LBB1_2: // %land.end +; CHECK-GI-NEXT: ret entry: %0 = load i32, ptr %pll, align 4 %cmp.not = icmp ugt i32 %0, %logLevel @@ -44,12 +58,20 @@ land.end: ; preds = %land.rhs, %entry declare i64 @llvm.ctlz.i64(i64 %in, i1) define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { -; CHECK-LABEL: lshr_ctlz_undef_cmpeq_one_i64: -; CHECK: // %bb.0: -; CHECK-NEXT: clz x8, x0 -; CHECK-NEXT: lsr x0, x8, #6 -; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: clz x8, x0 +; CHECK-SD-NEXT: lsr x0, x8, #6 +; CHECK-SD-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: clz x8, x0 +; CHECK-GI-NEXT: lsr x8, x8, #6 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1) %lshr = lshr i64 %ctlz, 6 %icmp = icmp eq i64 %lshr, 1 @@ -57,17 +79,30 @@ define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { } define i32 @PR17487(i1 %tobool) { -; CHECK-LABEL: PR17487: -; CHECK: // %bb.0: -; CHECK-NEXT: dup v0.2s, w0 -; CHECK-NEXT: mov w8, #1 // =0x1 -; CHECK-NEXT: dup v1.2d, x8 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-NEXT: bic v0.16b, v1.16b, v0.16b -; CHECK-NEXT: mov x8, v0.d[1] -; CHECK-NEXT: cmp x8, #1 -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: PR17487: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: dup v0.2s, w0 +; CHECK-SD-NEXT: mov w8, #1 // =0x1 +; CHECK-SD-NEXT: dup v1.2d, x8 +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-SD-NEXT: mov x8, v0.d[1] +; CHECK-SD-NEXT: cmp x8, #1 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: PR17487: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-GI-NEXT: mov v0.d[1], x0 +; CHECK-GI-NEXT: adrp x8, .LCPI3_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI3_0] +; CHECK-GI-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-GI-NEXT: mov d0, v0.d[1] +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret %tmp = insertelement <2 x i1> undef, i1 %tobool, i32 1 %tmp1 = zext <2 x i1> %tmp to <2 x i64> %tmp2 = xor <2 x i64> %tmp1, -- GitLab From b3fe27f2be0585d5d2ad46f96956ccfd76ca003e Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Sun, 24 Mar 2024 16:14:56 +0100 Subject: [PATCH 041/879] [InstCombine] Copy flags of extractelement for extelt -> icmp combine (#86366) Fixes #86164 --- llvm/include/llvm/IR/InstrTypes.h | 11 +++++++++++ llvm/lib/IR/Instructions.cpp | 10 ++++++++++ .../Transforms/InstCombine/InstCombineVectorOps.cpp | 4 +++- llvm/test/Transforms/InstCombine/scalarization.ll | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index e8c2cba8418d..d2b33fe9c651 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1058,6 +1058,17 @@ public: static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); + /// Construct a compare instruction, given the opcode, the predicate, + /// the two operands and the instruction to copy the flags from. Optionally + /// (if InstBefore is specified) insert the instruction into a BasicBlock + /// right before the specified instruction. The specified Instruction is + /// allowed to be a dereferenced end iterator. Create a CmpInst + static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name = "", + Instruction *InsertBefore = nullptr); + /// Get the opcode casted to the right type OtherOps getOpcode() const { return static_cast(Instruction::getOpcode()); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 494d50f89e37..8ca211e6e2e7 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -4623,6 +4623,16 @@ CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, S1, S2, Name); } +CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name, + Instruction *InsertBefore) { + CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore); + Inst->copyIRFlags(FlagsSource); + return Inst; +} + void CmpInst::swapOperands() { if (ICmpInst *IC = dyn_cast(this)) IC->swapOperands(); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c7f4fb17648c..99f1f8eb34bb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -487,7 +487,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index) Value *E0 = Builder.CreateExtractElement(X, Index); Value *E1 = Builder.CreateExtractElement(Y, Index); - return CmpInst::Create(cast(SrcVec)->getOpcode(), Pred, E0, E1); + CmpInst *SrcCmpInst = cast(SrcVec); + return CmpInst::CreateWithCopiedFlags(SrcCmpInst->getOpcode(), Pred, E0, E1, + SrcCmpInst); } if (auto *I = dyn_cast(SrcVec)) { diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll index fe6dc526bd50..7e645ef7e883 100644 --- a/llvm/test/Transforms/InstCombine/scalarization.ll +++ b/llvm/test/Transforms/InstCombine/scalarization.ll @@ -341,6 +341,17 @@ define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) { ret i1 %ext } +define i1 @extractelt_vector_fcmp_copy_flags(<4 x float> %x) { +; CHECK-LABEL: @extractelt_vector_fcmp_copy_flags( +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = fcmp nsz arcp oeq float [[TMP1]], 0.000000e+00 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %cmp = fcmp nsz arcp oeq <4 x float> %x, zeroinitializer + %r = extractelement <4 x i1> %cmp, i32 2 + ret i1 %r +} + define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) { ; ; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use( -- GitLab From 9632e1515c93453efc39752b1c9f32aedd358fbc Mon Sep 17 00:00:00 2001 From: houndlord <45179481+houndlord@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:33:16 +0100 Subject: [PATCH 042/879] Match fixed width ISD::AVGFLOORS + ISD::AVGCEILS patterns (#86222) --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++++++++++----- llvm/test/CodeGen/AArch64/hadd-combine.ll | 24 ++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index e27a8bb8fdac..05b4ce3aaa2c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2529,20 +2529,28 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); } -// Attempt to form avgceilu(A, B) from (A | B) - ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGCEILU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) { - SDValue A, B; if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) { + if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B); + } + } return SDValue(); } @@ -2837,20 +2845,29 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { return SDValue(); } -// Attempt to form avgflooru(A, B) from (A & B) + ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGFLOORU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) { - SDValue A, B; if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) { + if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B); + } + } + return SDValue(); } @@ -2869,8 +2886,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGFLOORU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGFLOORU(N, DAG)) + // Try to match AVGFLOOR fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG)) return V; // fold (a+b) -> (a|b) iff a and b share no bits. @@ -3868,8 +3885,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGCEILU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGCEILU(N, DAG)) + // Try to match AVGCEIL fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG)) return V; if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N))) diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll index e12502980790..491bf40ea4aa 100644 --- a/llvm/test/CodeGen/AArch64/hadd-combine.ll +++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll @@ -341,6 +341,18 @@ define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: srhadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %or = or <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = sub <8 x i16> %or, %srl + ret <8 x i16> %res +} + define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) { ; CHECK-LABEL: rhaddu_base: ; CHECK: // %bb.0: @@ -879,6 +891,18 @@ define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: shadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %and = and <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = add <8 x i16> %and, %srl + ret <8 x i16> %res +} + declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>) declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>) declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>) -- GitLab From 48048051323d5dd74057dc5f32df8c3c323afcd5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Sun, 24 Mar 2024 11:43:15 -0600 Subject: [PATCH 043/879] [lld/ELF][X86] Respect outSecOff when checking if GOTPCREL can be relaxed (#86334) The existing implementation didn't handle when the input text section was some offset from the output section. This resulted in an assert in relaxGot() with an lld built with asserts for some large binaries, or even worse, a silently broken binary with an lld without asserts. --- lld/ELF/Arch/X86_64.cpp | 7 ++++--- lld/test/ELF/x86-64-gotpc-relax-too-far.s | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index de459013595f..a85bf3aa0c09 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -328,9 +328,10 @@ bool X86_64::relaxOnce(int pass) const { if (rel.expr != R_RELAX_GOT_PC) continue; - uint64_t v = sec->getRelocTargetVA( - sec->file, rel.type, rel.addend, - sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr); + uint64_t v = sec->getRelocTargetVA(sec->file, rel.type, rel.addend, + sec->getOutputSection()->addr + + sec->outSecOff + rel.offset, + *rel.sym, rel.expr); if (isInt<32>(v)) continue; if (rel.sym->auxIdx == 0) { diff --git a/lld/test/ELF/x86-64-gotpc-relax-too-far.s b/lld/test/ELF/x86-64-gotpc-relax-too-far.s index 74aa6d8f65a0..ba41faab67de 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-too-far.s +++ b/lld/test/ELF/x86-64-gotpc-relax-too-far.s @@ -5,7 +5,10 @@ # RUN: llvm-objdump --no-print-imm-hex -d %t/bin | FileCheck --check-prefix=DISASM %s # RUN: llvm-readelf -S %t/bin | FileCheck --check-prefixes=GOT %s # RUN: ld.lld -T %t/lds2 %t/a.o -o %t/bin2 -# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=UNNECESSARY-GOT %s +# RUN: llvm-objdump --no-print-imm-hex -d %t/bin2 | FileCheck --check-prefix=DISASM %s +# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=GOT %s +# RUN: ld.lld -T %t/lds3 %t/a.o -o %t/bin3 +# RUN: llvm-readelf -S %t/bin3 | FileCheck --check-prefixes=UNNECESSARY-GOT %s # DISASM: <_foo>: # DISASM-NEXT: movl 2097146(%rip), %eax @@ -47,6 +50,13 @@ SECTIONS { data 0x80200000 : { *(data) } } #--- lds2 +SECTIONS { + .text.foo 0x100000 : { *(.text.foo) } + .text 0x1ff000 : { . = . + 0x1000 ; *(.text) } + .got 0x300000 : { *(.got) } + data 0x80200000 : { *(data) } +} +#--- lds3 SECTIONS { .text.foo 0x100000 : { *(.text.foo) } .text 0x200000 : { *(.text) } -- GitLab From d4a4585165f5c6ca8c42920b70e1b47696ff1172 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 24 Mar 2024 19:52:19 +0100 Subject: [PATCH 044/879] [clang-tidy] Ignore expresions in unevaluated context in bugprone-inc-dec-in-conditions (#85849) Skip checking for references to variable in unevaluated context, like decltype, static_assert and so on. Closes #85838 --- .../clang-tidy/bugprone/IncDecInConditionsCheck.cpp | 8 +++++++- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/bugprone/inc-dec-in-conditions.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index 16f43128d55e..9b3b01eb0268 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -31,6 +31,10 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), cxxOperatorCallExpr(isComparisonOperator()))); + auto IsInUnevaluatedContext = + expr(anyOf(hasAncestor(expr(matchers::hasUnevaluatedContext())), + hasAncestor(typeLoc()))); + Finder->addMatcher( expr( OperatorMatcher, unless(isExpansionInSystemHeader()), @@ -42,12 +46,14 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { cxxOperatorCallExpr( isPrePostOperator(), hasUnaryOperand(expr().bind("operand")))), + unless(IsInUnevaluatedContext), hasAncestor( expr(equalsBoundNode("parent"), hasDescendant( expr(unless(equalsBoundNode("operand")), matchers::isStatementIdenticalToBoundNode( - "operand")) + "operand"), + unless(IsInUnevaluatedContext)) .bind("second"))))) .bind("operator"))), this); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index a604e9276668..2392ccaf6575 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -139,6 +139,10 @@ Changes in existing checks ` check by detecting side effect from calling a method with non-const reference parameters. +- Improved :doc:`bugprone-inc-dec-in-conditions + ` check to ignore code + within unevaluated contexts, such as ``decltype``. + - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion ` check by eliminating false positives resulting from direct usage of bitwise operators diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp index 82af039973c3..91de013138f0 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp @@ -68,3 +68,13 @@ bool doubleCheck(Container x) { // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: decrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] // CHECK-MESSAGES: :[[@LINE-2]]:31: warning: incrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] } + +namespace PR85838 { + void test() + { + auto foo = 0; + auto bar = 0; + if (++foo < static_cast(bar)) {} + if (static_cast(bar) < foo) {} + } +} -- GitLab From 909ea28ac60760a0e9b39369b2f54a4e8f1daec9 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 14:29:23 -0500 Subject: [PATCH 045/879] [Libomptarget] Specificall add LLVM include dirs in plugins --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 0420d0e6f1f8..b84f3d7b137c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -60,6 +60,7 @@ target_link_options(PluginCommon PUBLIC ${offload_link_flags}) target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_LLVM_INCLUDE_DIRS} ${LIBOMPTARGET_INCLUDE_DIR} ) -- GitLab From 488a18738f5c275093bcc5459da69e9b4c9de074 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sun, 24 Mar 2024 10:41:21 -0500 Subject: [PATCH 046/879] [JITLink][ELF] Improve ELF section start/end symbol handling. This commit adds section start and stop symbol handling to ELF/aarch64, and fixes the section symbol prefixes (using `__start_` and `__stop_`, rather than `__start` and `__end`). It also adds a testcase for handling of these symbols. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 2 +- .../DefineExternalSectionStartAndEndSymbols.h | 19 ++++++++ .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 7 +++ .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 18 -------- .../ELF_section_start_and_stop_symbols.s | 43 +++++++++++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 30a9383dc0bc..4ce0c8d9fd17 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -567,7 +567,7 @@ public: orc::ExecutorAddrDiff getOffset() const { return Offset; } void setOffset(orc::ExecutorAddrDiff NewOffset) { - assert(NewOffset < getBlock().getSize() && "Offset out of range"); + assert(NewOffset <= getBlock().getSize() && "Offset out of range"); Offset = NewOffset; } diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 159880e4b152..aef441d093a2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -108,6 +108,25 @@ createDefineExternalSectionStartAndEndSymbolsPass( std::forward(F)); } +/// ELF section start/end symbol detection. +inline SectionRangeSymbolDesc +identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { + constexpr StringRef StartSymbolPrefix = "__start_"; + constexpr StringRef EndSymbolPrefix = "__stop_"; + + auto SymName = Sym.getName(); + if (SymName.starts_with(StartSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) + return {*Sec, true}; + } else if (SymName.starts_with(EndSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) + return {*Sec, false}; + } + return {}; +} + } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index f17b2c626ac2..4f64d83162b2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -20,6 +20,8 @@ #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" +#include "DefineExternalSectionStartAndEndSymbols.h" + #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -611,6 +613,11 @@ void link_ELF_aarch64(std::unique_ptr G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); + // Resolve any external section start / end symbols. + Config.PostAllocationPasses.push_back( + createDefineExternalSectionStartAndEndSymbolsPass( + identifyELFSectionStartAndEndSymbols)); + // Add an in-place GOT/TLS/Stubs build pass. Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index a1fe9c5fcd73..52dd83d9040f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -343,24 +343,6 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } -static SectionRangeSymbolDesc -identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { - constexpr StringRef StartSymbolPrefix = "__start"; - constexpr StringRef EndSymbolPrefix = "__end"; - - auto SymName = Sym.getName(); - if (SymName.starts_with(StartSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) - return {*Sec, true}; - } else if (SymName.starts_with(EndSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) - return {*Sec, false}; - } - return {}; -} - void link_ELF_x86_64(std::unique_ptr G, std::unique_ptr Ctx) { PassConfiguration Config; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s new file mode 100644 index 000000000000..f8e7ba96f006 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s @@ -0,0 +1,43 @@ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -position-independent \ +# RUN: -filetype=obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + + .text + .file "elf_section_start_stop.c" + .globl main + .p2align 2 + .type main,@function +main: + adrp x8, z + adrp x9, y + ldr w8, [x8, :lo12:z] + ldr w9, [x9, :lo12:y] + sub w0, w8, w9 + ret +.Lfunc_end0: + .size main, .Lfunc_end0-main + + .type x,@object + .section custom_section,"aw",@progbits + .globl x + .p2align 2 +x: + .word 42 + .size x, 4 + +# jitlink-check: *{8}z = (*{8}y) + 4 + + .type y,@object + .data + .globl y + .p2align 3, 0x0 +y: + .xword __start_custom_section + .size y, 8 + + .type z,@object + .globl z + .p2align 3, 0x0 +z: + .xword __stop_custom_section + .size z, 8 -- GitLab From 9f0321ccf118b37e5cb93cabd2acbf600c36b6ee Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 15:23:50 -0500 Subject: [PATCH 047/879] [Libomptarget] Make plugins depend explicitly on `intrinsics_gen` Summary: It's possible for the OpenMP offloading plugins to be build before tablegen is run despite the fact that we rely on it. Simply make it depend on it currently. --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index b84f3d7b137c..a7350e662a7c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(PluginCommon OBJECT src/RPC.cpp src/Utils/ELF.cpp ) +add_dependencies(PluginCommon intrinsics_gen) # Only enable JIT for those targets that LLVM can support. string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported) -- GitLab From 81e2693c1202d6c4e48dbf2d985153b03cfccb79 Mon Sep 17 00:00:00 2001 From: Hui Date: Sun, 24 Mar 2024 20:52:47 +0000 Subject: [PATCH 048/879] [libc++][test] Fix race condition in condition_variable_any tests (#84788) Some tests in `condition_variable_any` use two `shared_lock` to guard, which does not work. The fix is to make the writer to use `unique_lock` --- .../thread.condition.condvarany/wait_for_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_until_token_pred.pass.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp index 4ea60557d9f8..7a39d1253a33 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(2ms); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp index e96a3e8bd1bc..f322d8cfdc68 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp @@ -63,7 +63,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp index d649db025d75..e7388b9ce0e1 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); -- GitLab From f0a8738401137c32f3e1b9799244ed643feb7bf0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 24 Mar 2024 20:59:53 +0000 Subject: [PATCH 049/879] [VPlan] Generate CalculateTripCountMinusVF for Part 0 only. (NFCI). The value produced by CalculateTripCountMinusVF VPInstructions is independent of the part. Only compute it for part 0 and use that for other parts. --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 + .../AArch64/scalable-strict-fadd.ll | 63 +++---------------- .../AArch64/sve-tail-folding-unroll.ll | 42 ++----------- .../AArch64/uniform-args-call-variants.ll | 13 +--- 4 files changed, 21 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d75e322a74cf..f91997a37993 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -345,6 +345,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name); } case VPInstruction::CalculateTripCountMinusVF: { + if (Part != 0) + return State.get(this, 0, /*IsScalar*/ true); + Value *ScalarTC = State.get(getOperand(0), {0, 0}); Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index fc67fb5aded6..ad6e8534f318 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -403,21 +403,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -492,9 +477,9 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP78:%.*]] = mul i64 [[TMP77]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP79:%.*]] = add i64 [[INDEX]], [[TMP78]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP80:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP81:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP82:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1715,21 +1700,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -1826,9 +1796,9 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -2129,21 +2099,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -2240,9 +2195,9 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index 1a6e83a61ce7..2acc1ddbffea 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -25,21 +25,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -107,9 +92,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 12 ; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP72:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP73:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP74:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -167,21 +152,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -275,9 +245,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP92:%.*]] = mul i64 [[TMP91]], 12 ; CHECK-NEXT: [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP94:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP95:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT14]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP96:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT15]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 4957bbeda671..d8f14f30295b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -40,9 +40,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -71,7 +68,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -129,9 +126,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -160,7 +154,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -207,7 +201,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) -; INTERLEAVE-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1 ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -237,7 +230,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 ; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP0]] ; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: ; INTERLEAVE-NEXT: ret void -- GitLab From 18a49f03aa2b6bfeb073648b9eb75277a2386fc4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2024 14:07:09 -0700 Subject: [PATCH 050/879] [ELF] Merge relaIplt into relaDyn `relaIplt` was added so that IRELATIVE relocations are placed at the end of .rela.dyn (since https://reviews.llvm.org/D65651) or .rela.plt (--pack-dyn-relocs=android[+relr]). Unfortunately, handling `relaIplt` requires special cases all over the code base. We can extend partitionRels/computeRels to partition both RELATIVE and IRELATIVE relocations, rendering `relaIplt` unneeded. The change allows IRELATIVE relocations in the DT_ANDROID_REL[A] table (untested?!), which may be processed before other types of relocations. This seems acceptable for Bionic's DEFINE_IFUNC_FOR use cases. In addition, this change simplies changing .rel[a].dyn to a compact relocation format (CREL). SHF_INFO_LINK is removed from .rel[a].dyn with IRELATIVE relocations. (See https://reviews.llvm.org/D89828). --- lld/ELF/Relocations.cpp | 10 ++--- lld/ELF/SyntheticSections.cpp | 19 +++----- lld/ELF/SyntheticSections.h | 1 - lld/ELF/Writer.cpp | 50 ++++++--------------- lld/test/ELF/aarch64-gnu-ifunc.s | 3 +- lld/test/ELF/arm-gnu-ifunc.s | 3 +- lld/test/ELF/gnu-ifunc-i386.s | 3 +- lld/test/ELF/systemz-ifunc-nonpreemptible.s | 2 +- 8 files changed, 28 insertions(+), 63 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 92a1b9baaca3..33c50133bec4 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1620,12 +1620,8 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // relatively straightforward. We create a PLT entry in Iplt, which is // usually at the end of .plt, which makes an indirect call using a // matching GOT entry in igotPlt, which is usually at the end of .got.plt. - // The GOT entry is relocated using an IRELATIVE relocation in relaIplt, - // which is usually at the end of .rela.plt. Unlike most relocations in - // .rela.plt, which may be evaluated lazily without -z now, dynamic - // loaders evaluate IRELATIVE relocs eagerly, which means that for - // IRELATIVE relocs only, GOT-generating relocations can point directly to - // .got.plt without requiring a separate GOT entry. + // The GOT entry is relocated using an IRELATIVE relocation in relaDyn, + // which is usually at the end of .rela.dyn. // // - Despite the fact that an ifunc does not have a fixed value, compilers // that are not passed -fPIC will assume that they do, and will emit @@ -1665,7 +1661,7 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // section/value fixed. auto *directSym = makeDefined(cast(sym)); directSym->allocateAux(); - addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, + addPltEntry(*in.iplt, *in.igotPlt, *mainPart->relaDyn, target->iRelativeRel, *directSym); sym.allocateAux(); symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 10eda17f0cb3..f924756ddddf 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1267,15 +1267,12 @@ DynamicSection::DynamicSection() // The output section .rela.dyn may include these synthetic sections: // // - part.relaDyn -// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn // - in.relaPlt: this is included if a linker script places .rela.plt inside // .rela.dyn // // DT_RELASZ is the total size of the included sections. static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { size_t size = relaDyn.getSize(); - if (in.relaIplt->getParent() == relaDyn.getParent()) - size += in.relaIplt->getSize(); if (in.relaPlt->getParent() == relaDyn.getParent()) size += in.relaPlt->getSize(); return size; @@ -1372,9 +1369,7 @@ DynamicSection::computeContents() { if (!config->shared && !config->relocatable && !config->zRodynamic) addInt(DT_DEBUG, 0); - if (part.relaDyn->isNeeded() || - (in.relaIplt->isNeeded() && - part.relaDyn->getParent() == in.relaIplt->getParent())) { + if (part.relaDyn->isNeeded()) { addInSec(part.relaDyn->dynamicTag, *part.relaDyn); entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(*part.relaDyn)); @@ -1657,10 +1652,6 @@ void RelocationBaseSection::finalizeContents() { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.gotPlt->getParent()->sectionIndex; } - if (in.relaIplt.get() == this && in.igotPlt->getParent()) { - getParent()->flags |= ELF::SHF_INFO_LINK; - getParent()->info = in.igotPlt->getParent()->sectionIndex; - } } void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) { @@ -1674,6 +1665,11 @@ void RelocationBaseSection::computeRels() { SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); parallelForEach(relocs, [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); }); + + auto irelative = std::partition( + relocs.begin() + numRelativeRelocs, relocs.end(), + [t = target->iRelativeRel](auto &r) { return r.type != t; }); + // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset // is to make results easier to read. @@ -1682,7 +1678,7 @@ void RelocationBaseSection::computeRels() { parallelSort(relocs.begin(), nonRelative, [&](auto &a, auto &b) { return a.r_offset < b.r_offset; }); // Non-relative relocations are few, so don't bother with parallelSort. - llvm::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) { + llvm::sort(nonRelative, irelative, [&](auto &a, auto &b) { return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset); }); } @@ -3843,7 +3839,6 @@ void InStruct::reset() { ppc32Got2.reset(); ibtPlt.reset(); relaPlt.reset(); - relaIplt.reset(); shStrTab.reset(); strTab.reset(); symTab.reset(); diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index b41e69410054..fa21b80a5a5e 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -1358,7 +1358,6 @@ struct InStruct { std::unique_ptr ppc32Got2; std::unique_ptr ibtPlt; std::unique_ptr relaPlt; - std::unique_ptr relaIplt; std::unique_ptr shStrTab; std::unique_ptr strTab; std::unique_ptr symTab; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 4eca7b22e90b..40d617b7fdf3 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -449,8 +449,8 @@ template void elf::createSyntheticSections() { add(*part.dynamic); add(*part.dynStrTab); - add(*part.relaDyn); } + add(*part.relaDyn); if (config->relrPackDynRelocs) { part.relrDyn = std::make_unique>(threadCount); @@ -550,17 +550,6 @@ template void elf::createSyntheticSections() { /*threadCount=*/1); add(*in.relaPlt); - // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative - // relocations are processed last by the dynamic loader. We cannot place the - // iplt section in .rel.dyn when Android relocation packing is enabled because - // that would cause a section type mismatch. However, because the Android - // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired - // behaviour by placing the iplt section in .rel.plt. - in.relaIplt = std::make_unique>( - config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, - /*sort=*/false, /*threadCount=*/1); - add(*in.relaIplt); - if ((config->emachine == EM_386 || config->emachine == EM_X86_64) && (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) { in.ibtPlt = std::make_unique(); @@ -1071,20 +1060,18 @@ void PhdrEntry::add(OutputSection *sec) { sec->ptLoad = this; } -// The beginning and the ending of .rel[a].plt section are marked -// with __rel[a]_iplt_{start,end} symbols if it is a statically linked -// executable. The runtime needs these symbols in order to resolve -// all IRELATIVE relocs on startup. For dynamic executables, we don't -// need these symbols, since IRELATIVE relocs are resolved through GOT -// and PLT. For details, see http://www.airs.com/blog/archives/403. +// A statically linked position-dependent executable should only contain +// IRELATIVE relocations and no other dynamic relocations. Encapsulation symbols +// __rel[a]_iplt_{start,end} will be defined for .rel[a].dyn, to be +// processed by the libc runtime. Other executables or DSOs use dynamic tags +// instead. template void Writer::addRelIpltSymbols() { if (config->isPic) return; - // By default, __rela_iplt_{start,end} belong to a dummy section 0 - // because .rela.plt might be empty and thus removed from output. - // We'll override Out::elfHeader with In.relaIplt later when we are - // sure that .rela.plt exists in output. + // __rela_iplt_{start,end} are initially defined relative to dummy section 0. + // We'll override Out::elfHeader with relaDyn later when we are sure that + // .rela.dyn will be present in the output. ElfSym::relaIpltStart = addOptionalRegular( config->isRela ? "__rela_iplt_start" : "__rel_iplt_start", Out::elfHeader, 0, STV_HIDDEN); @@ -1110,11 +1097,11 @@ template void Writer::setReservedSymbolSections() { ElfSym::globalOffsetTable->section = sec; } - // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. - if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { - ElfSym::relaIpltStart->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->value = in.relaIplt->getSize(); + // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn. + if (ElfSym::relaIpltStart && mainPart->relaDyn->isNeeded()) { + ElfSym::relaIpltStart->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->value = mainPart->relaDyn->getSize(); } PhdrEntry *last = nullptr; @@ -1470,14 +1457,6 @@ static void sortSection(OutputSection &osec, if (name == ".init" || name == ".fini") return; - // IRelative relocations that usually live in the .rel[a].dyn section should - // be processed last by the dynamic loader. To achieve that we add synthetic - // sections in the required order from the beginning so that the in.relaIplt - // section is placed last in an output section. Here we just do not apply - // sorting for an output section which holds the in.relaIplt section. - if (in.relaIplt->getParent() == &osec) - return; - // Sort input sections by priority using the list provided by // --symbol-ordering-file or --shuffle-sections=. This is a least significant // digit radix sort. The sections may be sorted stably again by a more @@ -2196,7 +2175,6 @@ template void Writer::finalizeSections() { finalizeSynthetic(in.mipsGot.get()); finalizeSynthetic(in.igotPlt.get()); finalizeSynthetic(in.gotPlt.get()); - finalizeSynthetic(in.relaIplt.get()); finalizeSynthetic(in.relaPlt.get()); finalizeSynthetic(in.plt.get()); finalizeSynthetic(in.iplt.get()); diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s index dee24779d913..d76b54eabf8a 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc.s +++ b/lld/test/ELF/aarch64-gnu-ifunc.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_RELA // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0x158 // CHECK-NEXT: Size: 48 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 8 // CHECK-NEXT: EntrySize: 24 // CHECK-NEXT: } diff --git a/lld/test/ELF/arm-gnu-ifunc.s b/lld/test/ELF/arm-gnu-ifunc.s index 562478256fd8..d49ca18e991e 100644 --- a/lld/test/ELF/arm-gnu-ifunc.s +++ b/lld/test/ELF/arm-gnu-ifunc.s @@ -30,13 +30,12 @@ _start: // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: 0x100F4 // CHECK-NEXT: Offset: 0xF4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK: Name: .iplt // CHECK-NEXT: Type: SHT_PROGBITS // CHECK-NEXT: Flags [ diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s index b502fd6e9ae2..43b19b27ea4e 100644 --- a/lld/test/ELF/gnu-ifunc-i386.s +++ b/lld/test/ELF/gnu-ifunc-i386.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0xD4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 4 // CHECK-NEXT: EntrySize: 8 // CHECK-NEXT: } diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s index 5056db302ca1..892bbde8d9c7 100644 --- a/lld/test/ELF/systemz-ifunc-nonpreemptible.s +++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s @@ -10,7 +10,7 @@ # CHECK: Section Headers: # CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al # CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 -# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 AI 0 4 8 +# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 A 0 0 8 # CHECK-NEXT: [ 2] .text PROGBITS 0000000001001188 000188 00001c 00 AX 0 0 4 # CHECK-NEXT: [ 3] .iplt PROGBITS 00000000010011b0 0001b0 000040 00 AX 0 0 16 # CHECK-NEXT: [ 4] .got.plt PROGBITS 00000000010021f0 0001f0 000010 00 WA 0 0 8 -- GitLab From 67f2267ae006f35edfdf36567646403e61527d1b Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Sun, 24 Mar 2024 14:24:44 -0700 Subject: [PATCH 051/879] [M68k][NFC] Suppress warning on an unused variable when assertion is disabled NFC. --- llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp index 158393f02a24..786aa7bcb64e 100644 --- a/llvm/lib/Target/M68k/M68kISelLowering.cpp +++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp @@ -939,6 +939,7 @@ SDValue M68kTargetLowering::LowerFormalArguments( for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; assert(VA.getValNo() != LastVal && "Same value in different locations"); + (void)LastVal; LastVal = VA.getValNo(); -- GitLab From cceedc939a43c7c732a5888364251775bffc2dba Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sun, 24 Mar 2024 15:22:40 -0700 Subject: [PATCH 052/879] [clang-format] Fix a crash with AlignArrayOfStructures option (#86420) Fixes #86109. --- clang/lib/Format/WhitespaceManager.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index fef85abf79a3..710bf8d8a8ec 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -1491,7 +1491,7 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start, : Cell); // Go to the next non-comment and ensure there is a break in front const auto *NextNonComment = C.Tok->getNextNonComment(); - while (NextNonComment->is(tok::comma)) + while (NextNonComment && NextNonComment->is(tok::comma)) NextNonComment = NextNonComment->getNextNonComment(); auto j = i; while (j < End && Changes[j].Tok != NextNonComment) diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index cf8d6ab691d9..03005384a6f6 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -21100,7 +21100,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresRightAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", " @@ -21353,7 +21360,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresLeftAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", {\"x\", " -- GitLab From 230b1895c493c511c11541af3b5bc819887c82a8 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Sun, 24 Mar 2024 18:56:47 -0700 Subject: [PATCH 053/879] [mlir][spirv] Add folding for [S|U|LessThan[Equal] (#85435) Add missing constant propogation folder for [S|U]LessThan[Equal]. Implement additional folding when the operands are equal for all ops. Allows for constant folding in the IndexToSPIRV pass. Part of work #70704 --- .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td | 8 + .../SPIRV/IR/SPIRVCanonicalization.cpp | 82 ++++++++ .../SPIRV/Transforms/canonicalize.mlir | 176 ++++++++++++++++++ 3 files changed, 266 insertions(+) diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td index 3ee239d6e1e3..14d639bc26f2 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td @@ -716,6 +716,8 @@ def SPIRV_SLessThanOp : SPIRV_LogicalBinaryOp<"SLessThan", ``` }]; + + let hasFolder = 1; } // ----- @@ -745,6 +747,8 @@ def SPIRV_SLessThanEqualOp : SPIRV_LogicalBinaryOp<"SLessThanEqual", %5 = spirv.SLessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -886,6 +890,8 @@ def SPIRV_ULessThanOp : SPIRV_LogicalBinaryOp<"ULessThan", %5 = spirv.ULessThan %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -949,6 +955,8 @@ def SPIRV_ULessThanEqualOp : SPIRV_LogicalBinaryOp<"ULessThanEqual", %5 = spirv.ULessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } #endif // MLIR_DIALECT_SPIRV_IR_LOGICAL_OPS diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index ff4bace9a4d8..3f1a7826f8b6 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -880,6 +880,88 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) { }); } +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::SLessThanOp::fold(spirv::SLessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.slt(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::SLessThanEqualOp::fold(spirv::SLessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.sle(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ULessThanOp::fold(spirv::ULessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ult(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::ULessThanEqualOp::fold(spirv::ULessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ule(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + //===----------------------------------------------------------------------===// // spirv.ShiftLeftLogical //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index de21d114e9fc..ed0bd070c43c 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> { // ----- +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @slt_same +func.func @slt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.SLessThan %arg0, %arg0 : i32 + %1 = spirv.SLessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_slt +func.func @const_fold_scalar_slt() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThan %c5, %c6 : i32 + %1 = spirv.SLessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_slt +func.func @const_fold_vector_slt() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.SLessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @sle_same +func.func @sle_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.SLessThanEqual %arg0, %arg0 : i32 + %1 = spirv.SLessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_sle +func.func @const_fold_scalar_sle() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThanEqual %c5, %c6 : i32 + %1 = spirv.SLessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_sle +func.func @const_fold_vector_sle() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.SLessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ult_same +func.func @ult_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.ULessThan %arg0, %arg0 : i32 + %1 = spirv.ULessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ult +func.func @const_fold_scalar_ult() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThan %c5, %cn6 : i32 + %1 = spirv.ULessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ult +func.func @const_fold_vector_ult() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.ULessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ule_same +func.func @ule_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.ULessThanEqual %arg0, %arg0 : i32 + %1 = spirv.ULessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ule +func.func @const_fold_scalar_ule() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThanEqual %c5, %cn6 : i32 + %1 = spirv.ULessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ule +func.func @const_fold_vector_ule() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.ULessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.LeftShiftLogical //===----------------------------------------------------------------------===// -- GitLab From 7d2d8e2a7245e4e64da22cb3c422ea3be5a0bf0a Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Mon, 25 Mar 2024 10:59:42 +0900 Subject: [PATCH 054/879] [mlir][complex] Fastmath flag for the trigonometric ops in complex (#85563) Support Fastmath flag to convert trigonometric ops in the complex dialect. See: https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981 --- .../ComplexToStandard/ComplexToStandard.cpp | 50 +++++++++++-------- .../convert-to-standard.mlir | 46 +++++++++++++++++ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp index 76729278ec1b..17f64f1b65b7 100644 --- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp +++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp @@ -196,6 +196,7 @@ struct TrigonometricOpConversion : public OpConversionPattern { auto loc = op.getLoc(); auto type = cast(adaptor.getComplex().getType()); auto elementType = cast(type.getElementType()); + arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); Value real = rewriter.create(loc, elementType, adaptor.getComplex()); @@ -207,14 +208,14 @@ struct TrigonometricOpConversion : public OpConversionPattern { // implementation in the subclass to combine them. Value half = rewriter.create( loc, elementType, rewriter.getFloatAttr(elementType, 0.5)); - Value exp = rewriter.create(loc, imag); - Value scaledExp = rewriter.create(loc, half, exp); - Value reciprocalExp = rewriter.create(loc, half, exp); - Value sin = rewriter.create(loc, real); - Value cos = rewriter.create(loc, real); + Value exp = rewriter.create(loc, imag, fmf); + Value scaledExp = rewriter.create(loc, half, exp, fmf); + Value reciprocalExp = rewriter.create(loc, half, exp, fmf); + Value sin = rewriter.create(loc, real, fmf); + Value cos = rewriter.create(loc, real, fmf); auto resultPair = - combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter); + combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf); rewriter.replaceOpWithNewOp(op, type, resultPair.first, resultPair.second); @@ -223,15 +224,17 @@ struct TrigonometricOpConversion : public OpConversionPattern { virtual std::pair combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const = 0; + Value cos, ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const = 0; }; struct CosOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex cosine is defined as; // cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy))) // Plugging in: @@ -241,10 +244,12 @@ struct CosOpConversion : public TrigonometricOpConversion { // We get: // Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x // Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x - Value sum = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultReal = rewriter.create(loc, sum, cos); - Value diff = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultImag = rewriter.create(loc, diff, sin); + Value sum = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultReal = rewriter.create(loc, sum, cos, fmf); + Value diff = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultImag = rewriter.create(loc, diff, sin, fmf); return {resultReal, resultImag}; } }; @@ -813,9 +818,10 @@ struct NegOpConversion : public OpConversionPattern { struct SinOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex sine is defined as; // sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy))) // Plugging in: @@ -825,10 +831,12 @@ struct SinOpConversion : public TrigonometricOpConversion { // We get: // Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x // Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x - Value sum = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultReal = rewriter.create(loc, sum, sin); - Value diff = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultImag = rewriter.create(loc, diff, cos); + Value sum = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultReal = rewriter.create(loc, sum, sin, fmf); + Value diff = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultImag = rewriter.create(loc, diff, cos, fmf); return {resultReal, resultImag}; } }; diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir index 5918ff2e0f36..bac94aae6b74 100644 --- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir +++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir @@ -1834,3 +1834,49 @@ func.func @complex_sqrt_with_fmf(%arg: complex) -> complex { // CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32 // CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex // CHECK: return %[[VAR41]] : complex + +// ----- + +// CHECK-LABEL: func @complex_cos_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_cos_with_fmf(%arg: complex) -> complex { + %cos = complex.cos %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] + +// ----- + +// CHECK-LABEL: func @complex_sin_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_sin_with_fmf(%arg: complex) -> complex { + %cos = complex.sin %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] -- GitLab From 2e4e04c59043a645572cf548de5d9c333a5d6641 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Mon, 25 Mar 2024 10:06:12 +0800 Subject: [PATCH 055/879] [X86][BF16] Do not lower to VCVTNEPS2BF16 without AVX512VL (#86395) Fixes: #86305 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ++- llvm/test/CodeGen/X86/pr86305.ll | 74 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/X86/pr86305.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 35f756ea5e1d..9acbe17d0bca 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21512,7 +21512,9 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { } if (VT.getScalarType() == MVT::bf16) { - if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT)) + if (SVT.getScalarType() == MVT::f32 && + ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) return Op; return SDValue(); } @@ -21619,7 +21621,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op, SDLoc DL(Op); MVT SVT = Op.getOperand(0).getSimpleValueType(); - if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) { + if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) { SDValue Res; Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0)); Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res); diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll new file mode 100644 index 000000000000..79b42bb2532c --- /dev/null +++ b/llvm/test/CodeGen/X86/pr86305.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16 | FileCheck %s + +define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { +; CHECK-LABEL: add: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movq %rdx, %rbx +; CHECK-NEXT: movzwl (%rsi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm1 +; CHECK-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, (%rbx) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq + %a = load bfloat, ptr %pa + %b = load bfloat, ptr %pb + %add = fadd bfloat %a, %b + store bfloat %add, ptr %pc + ret void +} + +define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind { +; CHECK-LABEL: fptrunc_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %ebx +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %ebp +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r14d +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r15d +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %eax +; CHECK-NEXT: vmovd %r15d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r14d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq + %b = fptrunc <4 x float> %a to <4 x bfloat> + ret <4 x bfloat> %b +} -- GitLab From 5e5b6561029665e69e033cff4216fecb78302259 Mon Sep 17 00:00:00 2001 From: Sergei Barannikov Date: Mon, 25 Mar 2024 05:13:48 +0300 Subject: [PATCH 056/879] [MC] Make `MCParsedAsmOperand::getReg()` return `MCRegister` (#86444) --- .../llvm/MC/MCParser/MCParsedAsmOperand.h | 3 ++- .../llvm/MC/MCParser/MCTargetAsmParser.h | 4 +-- llvm/lib/MC/MCParser/MCTargetAsmParser.cpp | 6 +++++ .../AArch64/AsmParser/AArch64AsmParser.cpp | 2 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2 +- .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 2 +- .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 2 +- .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 2 +- .../Target/CSKY/AsmParser/CSKYAsmParser.cpp | 2 +- .../Hexagon/AsmParser/HexagonAsmParser.cpp | 2 +- .../Target/Lanai/AsmParser/LanaiAsmParser.cpp | 2 +- .../AsmParser/LoongArchAsmParser.cpp | 4 +-- .../Target/M68k/AsmParser/M68kAsmParser.cpp | 4 +-- .../MSP430/AsmParser/MSP430AsmParser.cpp | 2 +- .../Target/Mips/AsmParser/MipsAsmParser.cpp | 2 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 26 ++++++++++--------- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 +-- .../Target/Sparc/AsmParser/SparcAsmParser.cpp | 2 +- .../SystemZ/AsmParser/SystemZAsmParser.cpp | 2 +- llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp | 2 +- .../AsmParser/WebAssemblyAsmParser.cpp | 2 +- llvm/lib/Target/X86/AsmParser/X86Operand.h | 2 +- .../Xtensa/AsmParser/XtensaAsmParser.cpp | 2 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 2 +- 24 files changed, 46 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 0c9668904e82..27ecb7b85d22 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -15,6 +15,7 @@ namespace llvm { +class MCRegister; class raw_ostream; /// MCParsedAsmOperand - This abstract class represents a source-level assembly @@ -57,7 +58,7 @@ public: virtual bool isImm() const = 0; /// isReg - Is this a register operand? virtual bool isReg() const = 0; - virtual unsigned getReg() const = 0; + virtual MCRegister getReg() const = 0; /// isMem - Is this a memory operand? virtual bool isMem() const = 0; diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 7edd3f8ce490..49ce417e6fbb 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -514,9 +514,7 @@ public: /// by the tied-operands checks in the AsmMatcher. This method can be /// overridden to allow e.g. a sub- or super-register as the tied operand. virtual bool areEqualRegs(const MCParsedAsmOperand &Op1, - const MCParsedAsmOperand &Op2) const { - return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); - } + const MCParsedAsmOperand &Op2) const; // Return whether this parser uses assignment statements with equals tokens virtual bool equalIsAsmAssignment() { return true; }; diff --git a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp index 0db5fb36f795..665d92eb9a21 100644 --- a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp +++ b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp @@ -8,6 +8,7 @@ #include "llvm/MC/MCParser/MCTargetAsmParser.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCRegister.h" using namespace llvm; @@ -48,3 +49,8 @@ ParseStatus MCTargetAsmParser::parseDirective(AsmToken DirectiveID) { return ParseStatus::Failure; return ParseStatus::NoMatch; } + +bool MCTargetAsmParser::areEqualRegs(const MCParsedAsmOperand &Op1, + const MCParsedAsmOperand &Op2) const { + return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); +} diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index b807aaf76fdb..21643ebb4138 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -654,7 +654,7 @@ public: return Barrier.HasnXSModifier; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..4648df199c74 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -981,7 +981,7 @@ public: return Imm.Type; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isRegKind()); return Reg.RegNo; } diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 9cfdb15a0f43..2ad576ab1a9c 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -1002,7 +1002,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp index db4aa03437c6..383dfcc31117 100644 --- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp +++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp @@ -195,7 +195,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_Memri) && "Invalid access!"); return RegImm.Reg; diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 1688355f427c..9672ed009e9b 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -148,7 +148,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp index 4711e58bbed6..30bd3dcefa60 100644 --- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp +++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp @@ -400,7 +400,7 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index 864591d4eb95..092cccbcca9c 100644 --- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -245,7 +245,7 @@ public: /// getEndLoc - Get the location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp index ff3649b77e35..6ab1375b974e 100644 --- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp +++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp @@ -151,7 +151,7 @@ public: // getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isReg() && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp index cf163e4e1200..20284b18428b 100644 --- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp +++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp @@ -467,9 +467,9 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } const MCExpr *getImm() const { diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp index b2c0fda1ccc2..126176133dc0 100644 --- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp +++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp @@ -157,7 +157,7 @@ public: bool isDReg() const; bool isFPDReg() const; bool isFPCReg() const; - unsigned getReg() const override; + MCRegister getReg() const override; void addRegOperands(MCInst &Inst, unsigned N) const; static std::unique_ptr createMemOp(M68kMemOp MemOp, SMLoc Start, @@ -312,7 +312,7 @@ bool M68kOperand::isReg() const { return Kind == KindTy::MemOp && MemOp.Op == M68kMemOp::Kind::Reg; } -unsigned M68kOperand::getReg() const { +MCRegister M68kOperand::getReg() const { assert(isReg()); return MemOp.OuterReg; } diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp index 818a468612a5..2bc1a89ef59c 100644 --- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp +++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp @@ -183,7 +183,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Reg && "Invalid access!"); return Reg; } diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 9d6e8dc573a8..076e0a20cb97 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -1458,7 +1458,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { // As a special case until we sort out the definition of div/divu, accept // $0/$zero here so that MCK_ZERO works correctly. if (Kind == k_RegisterIndex && RegIdx.Index == 0 && diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index 8108cfa521c8..55978af38000 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -276,9 +276,11 @@ public: return TLSReg.Sym; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Not implemented"); } + + unsigned getRegNum() const { assert(isRegNumber() && "Invalid access!"); - return (unsigned) Imm.Val; + return (unsigned)Imm.Val; } unsigned getFpReg() const { @@ -459,22 +461,22 @@ public: void addRegGPRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegsNoR0[getRegNum()])); } void addRegG8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegs[getRegNum()])); } void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegsNoX0[getRegNum()])); } void addRegG8pRCOperands(MCInst &Inst, unsigned N) const { @@ -498,12 +500,12 @@ public: void addRegF4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegF8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegFpRCOperands(MCInst &Inst, unsigned N) const { @@ -513,12 +515,12 @@ public: void addRegVFRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VFRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VFRegs[getRegNum()])); } void addRegVRRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VRegs[getRegNum()])); } void addRegVSRCOperands(MCInst &Inst, unsigned N) const { @@ -538,12 +540,12 @@ public: void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegSPERCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(SPERegs[getReg()])); + Inst.addOperand(MCOperand::createReg(SPERegs[getRegNum()])); } void addRegACCRCOperands(MCInst &Inst, unsigned N) const { diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index cb2ba52390e2..5e594d6cad6b 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -977,9 +977,9 @@ public: return Imm.IsRV64; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } StringRef getSysReg() const { diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp index be4ec1e9dce2..67e2b9d7c997 100644 --- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp +++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp @@ -307,7 +307,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp index a58e8e0dfedf..f2c04215d12d 100644 --- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp +++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp @@ -227,7 +227,7 @@ public: bool isReg(RegisterKind RegKind) const { return Kind == KindReg && Reg.Kind == RegKind; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindReg && "Not a register"); return Reg.Num; } diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp index f9e30a3a9378..691fe8fe3aa4 100644 --- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp +++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp @@ -344,7 +344,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 3cc4d50271eb..020c0d6229d2 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -100,7 +100,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand { bool isReg() const override { return false; } bool isBrList() const { return Kind == BrList; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Assembly inspects a register operand"); return 0; } diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h index 641158cb351f..78669784dd03 100644 --- a/llvm/lib/Target/X86/AsmParser/X86Operand.h +++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h @@ -167,7 +167,7 @@ struct X86Operand final : public MCParsedAsmOperand { Tok.Length = Value.size(); } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNo; } diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp index 3f808298527f..1fa00af2111e 100644 --- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp +++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp @@ -244,7 +244,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 5df7990d8fc2..0a2b1cf0f9a6 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -2519,7 +2519,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) { // Check for register operands, including sub-classes. OS << " if (Operand.isReg()) {\n"; OS << " MatchClassKind OpKind;\n"; - OS << " switch (Operand.getReg()) {\n"; + OS << " switch (Operand.getReg().id()) {\n"; OS << " default: OpKind = InvalidMatchClass; break;\n"; for (const auto &RC : Info.RegisterClasses) OS << " case " << RC.first->getValueAsString("Namespace") -- GitLab From 6af6416e89de1f4656a145c4843226b468718434 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 11:44:16 +0800 Subject: [PATCH 057/879] [RISCV] Add a tune feature to disable stripping W suffix (#86255) We have a hidden option to disable it, but I'd like to make it a tune feature. For some implementations, instructions with W suffix would be less costly as they only perform on 32 bits data. Though we may lose some chances to compress. --- llvm/lib/Target/RISCV/RISCVFeatures.td | 4 ++ llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 2 +- llvm/test/CodeGen/RISCV/strip-w-suffix.ll | 74 +++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index f3e641e25018..6ef2289bb4be 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1226,6 +1226,10 @@ def TuneNoSinkSplatOperands "false", "Disable sink splat operands to enable .vx, .vf," ".wx, and .wf instructions">; +def TuneNoStripWSuffix + : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false", + "Disable strip W suffix">; + def TuneConditionalCompressedMoveFusion : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion", "true", "Enable branch+c.mv fusion">; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index dcf70e8cad64..39d420c2fbf0 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -672,7 +672,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII, const RISCVSubtarget &ST, MachineRegisterInfo &MRI) { - if (DisableStripWSuffix) + if (DisableStripWSuffix || !ST.enableStripWSuffix()) return false; bool MadeChange = false; diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll new file mode 100644 index 000000000000..4124b3d0d360 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=STRIP %s +; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=NO-STRIP %s + +define i32 @addiw(i32 %a) { +; STRIP-LABEL: addiw: +; STRIP: # %bb.0: +; STRIP-NEXT: lui a1, 1 +; STRIP-NEXT: addi a1, a1, -1 +; STRIP-NEXT: addw a0, a0, a1 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addiw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: lui a1, 1 +; NO-STRIP-NEXT: addiw a1, a1, -1 +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: ret + %ret = add i32 %a, 4095 + ret i32 %ret +} + +define i32 @addw(i32 %a, i32 %b) { +; STRIP-LABEL: addw: +; STRIP: # %bb.0: +; STRIP-NEXT: add a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %add = add i32 %a, %b + %ret = add i32 %add, 1024 + ret i32 %ret +} + +define i32 @mulw(i32 %a, i32 %b) { +; STRIP-LABEL: mulw: +; STRIP: # %bb.0: +; STRIP-NEXT: mul a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: mulw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: mulw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %mul = mul i32 %a, %b + %ret = add i32 %mul, 1024 + ret i32 %ret +} + +define i32 @slliw(i32 %a) { +; STRIP-LABEL: slliw: +; STRIP: # %bb.0: +; STRIP-NEXT: slli a0, a0, 1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: slliw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: slliw a0, a0, 1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %shl = shl i32 %a, 1 + %ret = add i32 %shl, 1024 + ret i32 %ret +} -- GitLab From d9746a6a5d523e21eee2c1b50c2f08aa19396965 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 12:42:59 +0800 Subject: [PATCH 058/879] [RISCV][NFC] Pass LMUL to copyPhysRegVector The opcode will be determined by LMUL. Reviewers: preames, lukel97, topperc Reviewed By: lukel97, topperc Pull Request: https://github.com/llvm/llvm-project/pull/84448 --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 52 ++++++++++++------------ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 4ca300f9151e..14c2d41e80f1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -299,36 +299,36 @@ void RISCVInstrInfo::copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF) const { + RISCVII::VLMUL LMul, unsigned NF) const { const TargetRegisterInfo *TRI = STI.getRegisterInfo(); - RISCVII::VLMUL LMul; + unsigned Opc; unsigned SubRegIdx; unsigned VVOpc, VIOpc; - switch (Opc) { + switch (LMul) { default: llvm_unreachable("Impossible LMUL for vector register copy."); - case RISCV::VMV1R_V: - LMul = RISCVII::LMUL_1; + case RISCVII::LMUL_1: + Opc = RISCV::VMV1R_V; SubRegIdx = RISCV::sub_vrm1_0; VVOpc = RISCV::PseudoVMV_V_V_M1; VIOpc = RISCV::PseudoVMV_V_I_M1; break; - case RISCV::VMV2R_V: - LMul = RISCVII::LMUL_2; + case RISCVII::LMUL_2: + Opc = RISCV::VMV2R_V; SubRegIdx = RISCV::sub_vrm2_0; VVOpc = RISCV::PseudoVMV_V_V_M2; VIOpc = RISCV::PseudoVMV_V_I_M2; break; - case RISCV::VMV4R_V: - LMul = RISCVII::LMUL_4; + case RISCVII::LMUL_4: + Opc = RISCV::VMV4R_V; SubRegIdx = RISCV::sub_vrm4_0; VVOpc = RISCV::PseudoVMV_V_V_M4; VIOpc = RISCV::PseudoVMV_V_I_M4; break; - case RISCV::VMV8R_V: + case RISCVII::LMUL_8: assert(NF == 1); - LMul = RISCVII::LMUL_8; + Opc = RISCV::VMV8R_V; SubRegIdx = RISCV::sub_vrm1_0; // There is no sub_vrm8_0. VVOpc = RISCV::PseudoVMV_V_V_M8; VIOpc = RISCV::PseudoVMV_V_I_M8; @@ -505,87 +505,87 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // VR->VR copies. if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1); return; } if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2); return; } if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4); return; } if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV8R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_8); return; } if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/2); return; } if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/2); return; } if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4, /*NF=*/2); return; } if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/3); return; } if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/3); return; } if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/4); return; } if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/4); return; } if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/5); return; } if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/6); return; } if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/7); return; } if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/8); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 8a312ee5e779..dd049fca0597 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -69,7 +69,7 @@ public: void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF = 1) const; + RISCVII::VLMUL LMul, unsigned NF = 1) const; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override; -- GitLab From 373e77b4c0ad9b0bf370f0e5a32a4100a5459d82 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 25 Mar 2024 13:08:56 +0800 Subject: [PATCH 059/879] [RISCV] Generalize (sub zext, zext) -> (sext (sub zext, zext)) to add (#86248) This generalizes the combine added in #82455 to other binary ops, beginning with adds in this patch. Because the two zext operands are always +ve when treated as signed, and we don't get any overflow since the add is carried out in at least N * 2 bits of the narrow type, the result of the add will always be +ve. So we can use a zext for the outer extend, unlike sub which may produce a -ve result from two +ve operands. Although we could still use sext for add, I plan to add support for other binary ops like mul in a later patch, but mul requires zext to be correct (because the maximum value will take up the full N * 2 bits). So I've opted to use zext here too for consistency. Alive2 proof: https://alive2.llvm.org/ce/z/PRNsUM --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 75 +++++++--- .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll | 32 ++--- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 128 ++++++++---------- 3 files changed, 125 insertions(+), 110 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5214595485ca..4ace50aa477a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12913,6 +12913,55 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); } +// add (zext, zext) -> zext (add (zext, zext)) +// sub (zext, zext) -> sext (sub (zext, zext)) +// +// where the sum of the extend widths match, and the the range of the bin op +// fits inside the width of the narrower bin op. (For profitability on rvv, we +// use a power of two for both inner and outer extend.) +// +// TODO: Extend this to other binary ops +static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) { + + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND) + return SDValue(); + if (!N0.hasOneUse() || !N1.hasOneUse()) + return SDValue(); + + SDValue Src0 = N0.getOperand(0); + SDValue Src1 = N1.getOperand(0); + EVT SrcVT = Src0.getValueType(); + if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) || + SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 || + SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); + EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); + + Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); + Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); + + // Src0 and Src1 are zero extended, so they're always positive if signed. + // + // sub can produce a negative from two positive operands, so it needs sign + // extended. Other nodes produce a positive from two positive operands, so + // zero extend instead. + unsigned OuterExtend = + N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + + return DAG.getNode( + OuterExtend, SDLoc(N), VT, + DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1)); +} + // Try to turn (add (xor bool, 1) -1) into (neg bool). static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); @@ -12950,6 +12999,8 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, return V; if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) return V; + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (add (select lhs, rhs, cc, 0, y), x) -> // (select lhs, rhs, cc, x, (add x, y)) @@ -13017,28 +13068,8 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, } } - // sub (zext, zext) -> sext (sub (zext, zext)) - // where the sum of the extend widths match, and the inner zexts - // add at least one bit. (For profitability on rvv, we use a - // power of two for both inner and outer extend.) - if (VT.isVector() && Subtarget.getTargetLowering()->isTypeLegal(VT) && - N0.getOpcode() == N1.getOpcode() && N0.getOpcode() == ISD::ZERO_EXTEND && - N0.hasOneUse() && N1.hasOneUse()) { - SDValue Src0 = N0.getOperand(0); - SDValue Src1 = N1.getOperand(0); - EVT SrcVT = Src0.getValueType(); - if (Subtarget.getTargetLowering()->isTypeLegal(SrcVT) && - SrcVT == Src1.getValueType() && SrcVT.getScalarSizeInBits() >= 8 && - SrcVT.getScalarSizeInBits() < VT.getScalarSizeInBits() / 2) { - LLVMContext &C = *DAG.getContext(); - EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); - EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); - Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); - Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); - return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, - DAG.getNode(ISD::SUB, SDLoc(N), NarrowVT, Src0, Src1)); - } - } + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (sub x, (select lhs, rhs, cc, 0, y)) -> // (select lhs, rhs, cc, x, (sub x, y)) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll index 57a72c639b33..bc0bf5dd76ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll @@ -385,12 +385,12 @@ define <32 x i64> @vwaddu_v32i64(ptr %x, ptr %y) nounwind { define <2 x i32> @vwaddu_v2i32_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -912,12 +912,12 @@ define <4 x i64> @crash(<4 x i16> %x, <4 x i16> %y) { define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -930,12 +930,12 @@ define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -948,12 +948,12 @@ define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i16(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: vle16.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i16>, ptr %x %b = load <2 x i16>, ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 66a7eea18be5..0a7051633a19 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -468,11 +468,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -555,10 +553,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -588,11 +586,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -675,10 +671,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -708,11 +704,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -795,10 +789,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -828,11 +822,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -915,10 +907,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -948,11 +940,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1035,10 +1025,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1068,11 +1058,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1155,10 +1143,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1188,11 +1176,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1275,10 +1261,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1308,11 +1294,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From babbdad15b8049a6a78087d15a163d897f07d320 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:23:40 +0100 Subject: [PATCH 060/879] [AMDGPU] Handle non-register operands for S_SUB/ADD_U64_PSEUDO (#86104) This pseudo uses SSrc_b64 so it allows both an immediate or a register, but the lowering crashed on immediate operands. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- .../CodeGen/AMDGPU/add_sub_u64_pseudos.mir | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 7f0cff72c186..d437f339a687 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4859,8 +4859,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( if (Subtarget->hasScalarAddSub64()) { unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) - .addReg(Src0.getReg()) - .addReg(Src1.getReg()); + .add(Src0) + .add(Src1); } else { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const TargetRegisterClass *BoolRC = TRI->getBoolRC(); diff --git a/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir new file mode 100644 index 000000000000..cba114c3568a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir @@ -0,0 +1,68 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX11 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX12 %s + +--- +name: reg_ops +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: reg_ops + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub0 + ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY2]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], [[COPY3]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: reg_ops + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], [[DEF1]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = IMPLICIT_DEF + %2:sreg_64 = S_ADD_U64_PSEUDO %0, %1, implicit-def $scc +... + +--- +name: lhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: lhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 6565, [[COPY]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 0, [[COPY1]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: lhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 6565, [[DEF]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO 6565, %0, implicit-def $scc +... + +--- +name: rhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: rhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 6565, implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], 0, implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: rhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], 6565 + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO %0, 6565, implicit-def $scc +... -- GitLab From fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:40:35 +0100 Subject: [PATCH 061/879] [RFC][TableGen] Restructure TableGen Source (#80847) Refactor of the llvm-tblgen source into: - a "Basic" library, which contains the bare minimum utilities to build `llvm-min-tablegen` - a "Common" library which contains all of the helpers for TableGen backends. Such helpers can be shared by more than one backend, and even unit tested (e.g. CodeExpander is, maybe we can add more over time) Fixes #80647 --- llvm/unittests/TableGen/CMakeLists.txt | 2 +- llvm/unittests/TableGen/CodeExpanderTest.cpp | 4 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/AsmWriterEmitter.cpp | 14 ++-- llvm/utils/TableGen/Basic/CMakeLists.txt | 21 ++++++ .../{ => Basic}/CodeGenIntrinsics.cpp | 0 .../TableGen/{ => Basic}/CodeGenIntrinsics.h | 0 .../TableGen/{ => Basic}/SDNodeProperties.cpp | 0 .../TableGen/{ => Basic}/SDNodeProperties.h | 0 .../{ => Basic}/SequenceToOffsetTable.h | 0 llvm/utils/TableGen/CMakeLists.txt | 67 ++++++++----------- llvm/utils/TableGen/CallingConvEmitter.cpp | 2 +- llvm/utils/TableGen/CodeEmitterGen.cpp | 10 +-- llvm/utils/TableGen/CodeGenMapTable.cpp | 4 +- .../TableGen/{ => Common}/AsmWriterInst.cpp | 0 .../TableGen/{ => Common}/AsmWriterInst.h | 0 llvm/utils/TableGen/Common/CMakeLists.txt | 48 +++++++++++++ .../{ => Common}/CodeGenDAGPatterns.cpp | 0 .../{ => Common}/CodeGenDAGPatterns.h | 4 +- .../TableGen/{ => Common}/CodeGenHwModes.cpp | 0 .../TableGen/{ => Common}/CodeGenHwModes.h | 0 .../{ => Common}/CodeGenInstAlias.cpp | 0 .../TableGen/{ => Common}/CodeGenInstAlias.h | 0 .../{ => Common}/CodeGenInstruction.cpp | 0 .../{ => Common}/CodeGenInstruction.h | 0 .../{ => Common}/CodeGenRegisters.cpp | 0 .../TableGen/{ => Common}/CodeGenRegisters.h | 0 .../TableGen/{ => Common}/CodeGenSchedule.cpp | 0 .../TableGen/{ => Common}/CodeGenSchedule.h | 0 .../TableGen/{ => Common}/CodeGenTarget.cpp | 0 .../TableGen/{ => Common}/CodeGenTarget.h | 2 +- .../TableGen/{ => Common}/DAGISelMatcher.cpp | 0 .../TableGen/{ => Common}/DAGISelMatcher.h | 0 .../{ => Common}/GlobalISel/CXXPredicates.cpp | 0 .../{ => Common}/GlobalISel/CXXPredicates.h | 0 .../{ => Common}/GlobalISel/CodeExpander.cpp | 0 .../{ => Common}/GlobalISel/CodeExpander.h | 0 .../{ => Common}/GlobalISel/CodeExpansions.h | 0 .../{ => Common}/GlobalISel/CombinerUtils.h | 0 .../GlobalISel}/GlobalISelMatchTable.cpp | 4 +- .../GlobalISel}/GlobalISelMatchTable.h | 2 +- .../GlobalISelMatchTableExecutorEmitter.cpp | 0 .../GlobalISelMatchTableExecutorEmitter.h | 2 +- .../{ => Common}/GlobalISel/MatchDataInfo.cpp | 0 .../{ => Common}/GlobalISel/MatchDataInfo.h | 0 .../{ => Common}/GlobalISel/Patterns.cpp | 4 +- .../{ => Common}/GlobalISel/Patterns.h | 0 .../TableGen/{ => Common}/InfoByHwMode.cpp | 0 .../TableGen/{ => Common}/InfoByHwMode.h | 0 .../TableGen/{ => Common}/OptEmitter.cpp | 0 llvm/utils/TableGen/{ => Common}/OptEmitter.h | 0 .../{ => Common}/PredicateExpander.cpp | 0 .../TableGen/{ => Common}/PredicateExpander.h | 0 .../{ => Common}/SubtargetFeatureInfo.cpp | 0 .../{ => Common}/SubtargetFeatureInfo.h | 0 llvm/utils/TableGen/{ => Common}/Types.cpp | 0 llvm/utils/TableGen/{ => Common}/Types.h | 0 .../{ => Common}/VarLenCodeEmitterGen.cpp | 15 ++--- .../{ => Common}/VarLenCodeEmitterGen.h | 0 llvm/utils/TableGen/CompressInstEmitter.cpp | 6 +- llvm/utils/TableGen/DAGISelEmitter.cpp | 8 +-- llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/DAGISelMatcherGen.cpp | 14 ++-- llvm/utils/TableGen/DAGISelMatcherOpt.cpp | 6 +- llvm/utils/TableGen/DFAEmitter.cpp | 2 +- llvm/utils/TableGen/DFAPacketizerEmitter.cpp | 4 +- llvm/utils/TableGen/DXILEmitter.cpp | 4 +- llvm/utils/TableGen/DecoderEmitter.cpp | 10 +-- llvm/utils/TableGen/DisassemblerEmitter.cpp | 2 +- llvm/utils/TableGen/FastISelEmitter.cpp | 10 +-- llvm/utils/TableGen/GlobalISel/CMakeLists.txt | 20 ------ .../TableGen/GlobalISelCombinerEmitter.cpp | 24 +++---- llvm/utils/TableGen/GlobalISelEmitter.cpp | 18 ++--- llvm/utils/TableGen/InstrDocsEmitter.cpp | 6 +- llvm/utils/TableGen/InstrInfoEmitter.cpp | 16 ++--- llvm/utils/TableGen/IntrinsicEmitter.cpp | 4 +- .../TableGen/MacroFusionPredicatorEmitter.cpp | 4 +- llvm/utils/TableGen/OptParserEmitter.cpp | 2 +- llvm/utils/TableGen/OptRSTEmitter.cpp | 2 +- llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 4 +- llvm/utils/TableGen/RegisterBankEmitter.cpp | 6 +- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 12 ++-- .../utils/TableGen/SearchableTableEmitter.cpp | 4 +- llvm/utils/TableGen/SubtargetEmitter.cpp | 8 +-- .../WebAssemblyDisassemblerEmitter.cpp | 2 +- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86MnemonicTables.cpp | 4 +- llvm/utils/TableGen/X86RecognizableInstr.h | 2 +- 89 files changed, 238 insertions(+), 203 deletions(-) create mode 100644 llvm/utils/TableGen/Basic/CMakeLists.txt rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.h (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.h (100%) rename llvm/utils/TableGen/{ => Basic}/SequenceToOffsetTable.h (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.cpp (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.h (100%) create mode 100644 llvm/utils/TableGen/Common/CMakeLists.txt rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.h (99%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.h (99%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.cpp (100%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpansions.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CombinerUtils.h (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.cpp (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.h (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.h (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.cpp (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.h (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.cpp (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.h (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.h (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/Types.cpp (100%) rename llvm/utils/TableGen/{ => Common}/Types.h (100%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.cpp (97%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.h (100%) delete mode 100644 llvm/utils/TableGen/GlobalISel/CMakeLists.txt diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt index 7830e0218045..fae0eee06e4b 100644 --- a/llvm/unittests/TableGen/CMakeLists.txt +++ b/llvm/unittests/TableGen/CMakeLists.txt @@ -15,4 +15,4 @@ add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB ParserEntryPointTest.cpp ) -target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen) +target_link_libraries(TableGenTests PRIVATE LLVMTableGenCommon LLVMTableGen) diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp index 4a9a0e8c114b..1528884ffdf6 100644 --- a/llvm/unittests/TableGen/CodeExpanderTest.cpp +++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 0a2b1cf0f9a6..8b82ce899a48 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -95,12 +95,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SubtargetFeatureInfo.h" -#include "Types.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp index a27061ee585a..16661cd29edc 100644 --- a/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -11,13 +11,13 @@ // //===----------------------------------------------------------------------===// -#include "AsmWriterInst.h" -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/AsmWriterInst.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt new file mode 100644 index 000000000000..f2927d05c175 --- /dev/null +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -0,0 +1,21 @@ +# The basic TableGen library contains as little dependencies as possible. +# In particular, it does not depend on vt_gen -> it does not use ValueTypes. +# +# This library is the only thing included in `llvm-min-tablegen`. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL + CodeGenIntrinsics.cpp + SDNodeProperties.cpp +) +set_target_properties(LLVMTableGenBasic PROPERTIES FOLDER "Tablegenning") + +# Users may include its headers as "Basic/*.h" +target_include_directories(LLVMTableGenBasic + INTERFACE + $ + ) diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.cpp rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.h rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.h diff --git a/llvm/utils/TableGen/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.cpp rename to llvm/utils/TableGen/Basic/SDNodeProperties.cpp diff --git a/llvm/utils/TableGen/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.h rename to llvm/utils/TableGen/Basic/SDNodeProperties.h diff --git a/llvm/utils/TableGen/SequenceToOffsetTable.h b/llvm/utils/TableGen/Basic/SequenceToOffsetTable.h similarity index 100% rename from llvm/utils/TableGen/SequenceToOffsetTable.h rename to llvm/utils/TableGen/Basic/SequenceToOffsetTable.h diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 0100bf345ec2..14690329cabf 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -1,26 +1,25 @@ -add_subdirectory(GlobalISel) +# Basic utilities which is the strict minimum needed to build +# llvm-min-tblgen. +add_subdirectory(Basic) +# Common utilities are all of the reusable components and helper +# code needed by the backends. +add_subdirectory(Common) -add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL +set(LLVM_LINK_COMPONENTS Support) + +# llvm-min-tablegen only contains a subset of backends necessary to +# build llvm/include. It must not depend on TableGenCommon, as +# TableGenCommon depends on this already to generate things such as +# ValueType definitions. +add_tablegen(llvm-min-tblgen LLVM_HEADERS + TableGen.cpp Attributes.cpp - CodeGenIntrinsics.cpp DirectiveEmitter.cpp IntrinsicEmitter.cpp RISCVTargetDefEmitter.cpp - SDNodeProperties.cpp VTEmitter.cpp - PARTIAL_SOURCES_INTENDED - - LINK_COMPONENTS - Support - TableGen - ) -set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") + $ -set(LLVM_LINK_COMPONENTS Support) - -add_tablegen(llvm-min-tblgen LLVM_HEADERS - TableGen.cpp - $ PARTIAL_SOURCES_INTENDED ) set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning") @@ -35,63 +34,51 @@ add_tablegen(llvm-tblgen LLVM EXPORT LLVM AsmMatcherEmitter.cpp AsmWriterEmitter.cpp - AsmWriterInst.cpp - CTagsEmitter.cpp + Attributes.cpp CallingConvEmitter.cpp CodeEmitterGen.cpp - CodeGenDAGPatterns.cpp - CodeGenHwModes.cpp - CodeGenInstAlias.cpp - CodeGenInstruction.cpp CodeGenMapTable.cpp - CodeGenRegisters.cpp - CodeGenSchedule.cpp - CodeGenTarget.cpp + CompressInstEmitter.cpp + CTagsEmitter.cpp DAGISelEmitter.cpp DAGISelMatcherEmitter.cpp DAGISelMatcherGen.cpp DAGISelMatcherOpt.cpp - DAGISelMatcher.cpp DecoderEmitter.cpp DFAEmitter.cpp DFAPacketizerEmitter.cpp + DirectiveEmitter.cpp DisassemblerEmitter.cpp DXILEmitter.cpp ExegesisEmitter.cpp FastISelEmitter.cpp GlobalISelCombinerEmitter.cpp GlobalISelEmitter.cpp - GlobalISelMatchTable.cpp - GlobalISelMatchTableExecutorEmitter.cpp - InfoByHwMode.cpp - InstrInfoEmitter.cpp InstrDocsEmitter.cpp - OptEmitter.cpp + InstrInfoEmitter.cpp + IntrinsicEmitter.cpp + MacroFusionPredicatorEmitter.cpp OptParserEmitter.cpp OptRSTEmitter.cpp - PredicateExpander.cpp PseudoLoweringEmitter.cpp - CompressInstEmitter.cpp - MacroFusionPredicatorEmitter.cpp RegisterBankEmitter.cpp RegisterInfoEmitter.cpp + RISCVTargetDefEmitter.cpp SearchableTableEmitter.cpp SubtargetEmitter.cpp - SubtargetFeatureInfo.cpp TableGen.cpp - Types.cpp - VarLenCodeEmitterGen.cpp - X86DisassemblerTables.cpp + VTEmitter.cpp + WebAssemblyDisassemblerEmitter.cpp X86CompressEVEXTablesEmitter.cpp + X86DisassemblerTables.cpp X86FoldTablesEmitter.cpp X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp - WebAssemblyDisassemblerEmitter.cpp $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel) +target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp index 3c3a2874ce80..ec6ef56a66fa 100644 --- a/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp index 9194c13ccdcb..a57885f22d7e 100644 --- a/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -22,11 +22,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "VarLenCodeEmitterGen.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp index 03af0b49ba97..fbf1d47c0327 100644 --- a/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -75,8 +75,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" using namespace llvm; diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/Common/AsmWriterInst.cpp similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.cpp rename to llvm/utils/TableGen/Common/AsmWriterInst.cpp diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/Common/AsmWriterInst.h similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.h rename to llvm/utils/TableGen/Common/AsmWriterInst.h diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt new file mode 100644 index 000000000000..491d9bd2949d --- /dev/null +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -0,0 +1,48 @@ +# The common library is similar to the basic library except it can +# depend on vt_gen. +# +# This library contains the bulk of the supporting code for all +# TableGen backends. It's split off as a separate library to +# allow unit-testing those components. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL + GlobalISel/CodeExpander.cpp + GlobalISel/CXXPredicates.cpp + GlobalISel/GlobalISelMatchTable.cpp + GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp + GlobalISel/MatchDataInfo.cpp + GlobalISel/Patterns.cpp + + AsmWriterInst.cpp + CodeGenDAGPatterns.cpp + CodeGenHwModes.cpp + CodeGenInstAlias.cpp + CodeGenInstruction.cpp + CodeGenRegisters.cpp + CodeGenSchedule.cpp + CodeGenTarget.cpp + DAGISelMatcher.cpp + InfoByHwMode.cpp + OptEmitter.cpp + PredicateExpander.cpp + SubtargetFeatureInfo.cpp + Types.cpp + VarLenCodeEmitterGen.cpp + $ + + DEPENDS + vt_gen + ) +set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") +target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) + +# Users may include its headers as "Common/*.h" +target_include_directories(LLVMTableGenCommon + PUBLIC + $ + ) diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenDAGPatterns.cpp rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h similarity index 99% rename from llvm/utils/TableGen/CodeGenDAGPatterns.h rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.h index 823c40c922cb..7fcd39a9e940 100644 --- a/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h @@ -14,9 +14,9 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H -#include "CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SDNodeProperties.h" #include "CodeGenTarget.h" -#include "SDNodeProperties.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.cpp rename to llvm/utils/TableGen/Common/CodeGenHwModes.cpp diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/Common/CodeGenHwModes.h similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.h rename to llvm/utils/TableGen/Common/CodeGenHwModes.h diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.cpp rename to llvm/utils/TableGen/Common/CodeGenInstAlias.cpp diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/Common/CodeGenInstAlias.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.h rename to llvm/utils/TableGen/Common/CodeGenInstAlias.h diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.cpp rename to llvm/utils/TableGen/Common/CodeGenInstruction.cpp diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.h rename to llvm/utils/TableGen/Common/CodeGenInstruction.h diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.cpp rename to llvm/utils/TableGen/Common/CodeGenRegisters.cpp diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.h rename to llvm/utils/TableGen/Common/CodeGenRegisters.h diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.cpp rename to llvm/utils/TableGen/Common/CodeGenSchedule.cpp diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.h rename to llvm/utils/TableGen/Common/CodeGenSchedule.h diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenTarget.cpp rename to llvm/utils/TableGen/Common/CodeGenTarget.cpp diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h similarity index 99% rename from llvm/utils/TableGen/CodeGenTarget.h rename to llvm/utils/TableGen/Common/CodeGenTarget.h index e109c717dc01..df4c22ebb379 100644 --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/Common/CodeGenTarget.h @@ -16,10 +16,10 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#include "Basic/SDNodeProperties.h" #include "CodeGenHwModes.h" #include "CodeGenInstruction.h" #include "InfoByHwMode.h" -#include "SDNodeProperties.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.cpp rename to llvm/utils/TableGen/Common/DAGISelMatcher.cpp diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.h rename to llvm/utils/TableGen/Common/DAGISelMatcher.h diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.h rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpansions.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CombinerUtils.h rename to llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 45fb41b89f27..193f95443b16 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include "GlobalISelMatchTable.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" #include "llvm/Support/LEB128.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index b1ab7da8db44..aa86fad763d1 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H -#include "CodeGenDAGPatterns.h" +#include "Common/CodeGenDAGPatterns.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h index 7e952d6df309..d2b6a74c7577 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h @@ -15,7 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H -#include "SubtargetFeatureInfo.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.h rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISel/Patterns.cpp rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp index 758eac2dfebd..388bf7e9e833 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp @@ -7,11 +7,11 @@ //===----------------------------------------------------------------------===// #include "Patterns.h" -#include "../CodeGenInstruction.h" -#include "../CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" #include "CXXPredicates.h" #include "CodeExpander.h" #include "CodeExpansions.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/Patterns.h rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.h diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.cpp rename to llvm/utils/TableGen/Common/InfoByHwMode.cpp diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.h rename to llvm/utils/TableGen/Common/InfoByHwMode.h diff --git a/llvm/utils/TableGen/OptEmitter.cpp b/llvm/utils/TableGen/Common/OptEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/OptEmitter.cpp rename to llvm/utils/TableGen/Common/OptEmitter.cpp diff --git a/llvm/utils/TableGen/OptEmitter.h b/llvm/utils/TableGen/Common/OptEmitter.h similarity index 100% rename from llvm/utils/TableGen/OptEmitter.h rename to llvm/utils/TableGen/Common/OptEmitter.h diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.cpp rename to llvm/utils/TableGen/Common/PredicateExpander.cpp diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/Common/PredicateExpander.h similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.h rename to llvm/utils/TableGen/Common/PredicateExpander.h diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.cpp rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.h similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.h rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.h diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Common/Types.cpp similarity index 100% rename from llvm/utils/TableGen/Types.cpp rename to llvm/utils/TableGen/Common/Types.cpp diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Common/Types.h similarity index 100% rename from llvm/utils/TableGen/Types.h rename to llvm/utils/TableGen/Common/Types.h diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp similarity index 97% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.cpp rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp index bfb7e5c33317..4263d8f41715 100644 --- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp +++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp @@ -337,8 +337,8 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits, return; } - IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " - << "/*Index*/" << Index << "},"; + IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " << "/*Index*/" + << Index << "},"; SS.indent(4); for (unsigned I = 0; I < Bits.getNumWords(); ++I, ++Index) @@ -371,8 +371,8 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( if (ModeIt == InstIt->second.end()) ModeIt = InstIt->second.find(Universal); if (ModeIt == InstIt->second.end()) { - IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" - << "// " << R->getName() << " no encoding\n"; + IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" << "// " << R->getName() + << " no encoding\n"; continue; } const VarLenInst &VLI = ModeIt->second; @@ -492,10 +492,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n"; - SS.indent(I) << "Inst.insertBits(" - << "Scratch.extractBits(" << utostr(NumBits) << ", " - << utostr(LoBit) << ")" - << ", " << Offset << ");\n"; + SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits(" + << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", " + << Offset << ");\n"; HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit); } diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h similarity index 100% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.h rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp index f703fff0ef3e..fcf77934faac 100644 --- a/llvm/utils/TableGen/CompressInstEmitter.cpp +++ b/llvm/utils/TableGen/CompressInstEmitter.cpp @@ -64,9 +64,9 @@ //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp index 336cee09b90c..b43a8e659dd9 100644 --- a/llvm/utils/TableGen/DAGISelEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 533b8c423690..dcecac4380ce 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -10,12 +10,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index e8bdabaa0c7e..99babdf07316 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "InfoByHwMode.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 047d285f9914..224102e49d98 100644 --- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp index ce8cc2a078d7..567184d3d5ee 100644 --- a/llvm/utils/TableGen/DFAEmitter.cpp +++ b/llvm/utils/TableGen/DFAEmitter.cpp @@ -21,7 +21,7 @@ //===----------------------------------------------------------------------===// #include "DFAEmitter.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/UniqueVector.h" diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp index 26ea1846ffae..3c74df048660 100644 --- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp +++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -14,8 +14,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" #include "DFAEmitter.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index af1efb8aa99f..47c89df35e19 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 732f34ed04c5..494dc93faace 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "TableGenBackends.h" -#include "VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/CachedHashString.h" diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp index 2d653af4d302..d41750075b41 100644 --- a/llvm/utils/TableGen/DisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "TableGenBackends.h" #include "WebAssemblyDisassemblerEmitter.h" #include "X86DisassemblerTables.h" diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp index f04c6e3b3bf0..acfdc20316b7 100644 --- a/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/llvm/utils/TableGen/FastISelEmitter.cpp @@ -16,11 +16,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt deleted file mode 100644 index 7262c4058399..000000000000 --- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(LLVM_LINK_COMPONENTS - Support - TableGen - ) - -add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB - CodeExpander.cpp - CXXPredicates.cpp - MatchDataInfo.cpp - Patterns.cpp - - DEPENDS - vt_gen - ) - -# Users may include its headers as "GlobalISel/*.h" -target_include_directories(LLVMTableGenGlobalISel - INTERFACE - $ - ) diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index dee3cb4d71a4..39b9f8a2ae17 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -26,18 +26,18 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" -#include "GlobalISel/CXXPredicates.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISel/MatchDataInfo.h" -#include "GlobalISel/Patterns.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/CXXPredicates.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CombinerUtils.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/GlobalISel/MatchDataInfo.h" +#include "Common/GlobalISel/Patterns.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/Hashing.h" diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index e86057422cd7..25e302ce1ca4 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -30,15 +30,15 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "InfoByHwMode.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/InfoByHwMode.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGenTypes/LowLevelType.h" #include "llvm/CodeGenTypes/MachineValueType.h" diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp index efabf6bb7ba6..f948540e18db 100644 --- a/llvm/utils/TableGen/InstrDocsEmitter.cpp +++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -18,9 +18,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp index 2d08447429d9..36f8fa146539 100644 --- a/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -11,15 +11,15 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" -#include "SequenceToOffsetTable.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "TableGenBackends.h" -#include "Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 50a34eac7ca3..a7e99fa4c050 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "SequenceToOffsetTable.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index 91c3b0b4359c..e9e63fa8d0de 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -38,8 +38,8 @@ // //===---------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp index c25f6c59cab3..6334af53f88f 100644 --- a/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/llvm/utils/TableGen/OptParserEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp index 5a7f079dc168..75b7cbdf2988 100644 --- a/llvm/utils/TableGen/OptRSTEmitter.cpp +++ b/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp index 7f692f29192d..01cfd4a1d982 100644 --- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp +++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp index 8b59411c5bc3..5546e727af38 100644 --- a/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/BitVector.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index c4fc1930488c..a1259bff6ba8 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp index 51f18f360ed3..48ee23db957d 100644 --- a/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp index d350d7de139f..2e2c57b802ee 100644 --- a/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp index 928129f24fcb..e9436ab16e44 100644 --- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -14,7 +14,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyDisassemblerEmitter.h" -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 0a9abbfe186e..c721502a395f 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -11,8 +11,8 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 1319042e48d0..5871e678b16e 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FormattedStream.h" diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp index aeafee157462..d9ceed40f7c7 100644 --- a/llvm/utils/TableGen/X86MnemonicTables.cpp +++ b/llvm/utils/TableGen/X86MnemonicTables.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 68af68fb5aa0..12fb41750cb3 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/Support/X86DisassemblerDecoderCommon.h" #include #include -- GitLab From aa962d67ee896f416e285a9298e45fc08ff95eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:42:41 +0100 Subject: [PATCH 062/879] [clang-repl] Fix Value for platforms where unqualified char is unsigned (#86118) Signedness of unqualified `char` is unspecified and varies between platforms. This patch adds `Char_U` in `REPL_BUILTIN_TYPES` to account for platforms that default to `unsigned char`. --- clang/include/clang/Interpreter/Value.h | 1 + clang/unittests/Interpreter/InterpreterTest.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h index c380cd91550d..d70e8f871902 100644 --- a/clang/include/clang/Interpreter/Value.h +++ b/clang/include/clang/Interpreter/Value.h @@ -76,6 +76,7 @@ class QualType; X(bool, Bool) \ X(char, Char_S) \ X(signed char, SChar) \ + X(unsigned char, Char_U) \ X(unsigned char, UChar) \ X(short, Short) \ X(unsigned short, UShort) \ diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index e76c0677db5e..69bc2da24288 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -340,6 +340,12 @@ TEST(InterpreterTest, Value) { EXPECT_EQ(V1.getKind(), Value::K_Int); EXPECT_FALSE(V1.isManuallyAlloc()); + Value V1b; + llvm::cantFail(Interp->ParseAndExecute("char c = 42;")); + llvm::cantFail(Interp->ParseAndExecute("c", &V1b)); + EXPECT_TRUE(V1b.getKind() == Value::K_Char_S || + V1b.getKind() == Value::K_Char_U); + Value V2; llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;")); llvm::cantFail(Interp->ParseAndExecute("y", &V2)); -- GitLab From 0cf4788d9d0df60980cb48d28aafe7a86aa15a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:44:25 +0100 Subject: [PATCH 063/879] [clang-repl] Factor out CreateJITBuilder() and allow specialization in derived classes (#84461) The LLJITBuilder interface provides a very convenient way to configure the ORCv2 JIT engine. IncrementalExecutor already used it internally to construct the JIT, but didn't provide external access. This patch lifts control of the creation process to the Interpreter and allows injection of a custom instance through the extended interface. The Interpreter's default behavior remains unchanged and the IncrementalExecutor remains an implementation detail. --- clang/include/clang/Interpreter/Interpreter.h | 8 ++ clang/lib/Interpreter/IncrementalExecutor.cpp | 33 ++--- clang/lib/Interpreter/IncrementalExecutor.h | 9 +- clang/lib/Interpreter/Interpreter.cpp | 26 +++- .../Interpreter/InterpreterExtensionsTest.cpp | 121 +++++++++++++++++- 5 files changed, 175 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 1dcba1ef9679..970e0245417b 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -30,6 +30,7 @@ namespace llvm { namespace orc { class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -127,6 +128,13 @@ protected: // custom runtime. virtual std::unique_ptr FindRuntimeInterface(); + // Lazily construct thev ORCv2 JITBuilder. This called when the internal + // IncrementalExecutor is created. The default implementation populates an + // in-process JIT with debugging support. Override this to configure the JIT + // engine used for execution. + virtual llvm::Expected> + CreateJITBuilder(CompilerInstance &CI); + public: virtual ~Interpreter(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 40bcef94797d..6f036107c14a 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" @@ -36,26 +37,28 @@ LLVM_ATTRIBUTE_USED void linkComponents() { namespace clang { +llvm::Expected> +IncrementalExecutor::createDefaultJITBuilder( + llvm::orc::JITTargetMachineBuilder JTMB) { + auto JITBuilder = std::make_unique(); + JITBuilder->setJITTargetMachineBuilder(std::move(JTMB)); + JITBuilder->setPrePlatformSetup([](llvm::orc::LLJIT &J) { + // Try to enable debugging of JIT'd code (only works with JITLink for + // ELF and MachO). + consumeError(llvm::orc::enableDebuggerSupport(J)); + return llvm::Error::success(); + }); + return std::move(JITBuilder); +} + IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, - llvm::Error &Err, - const clang::TargetInfo &TI) + llvm::orc::LLJITBuilder &JITBuilder, + llvm::Error &Err) : TSCtx(TSC) { using namespace llvm::orc; llvm::ErrorAsOutParameter EAO(&Err); - auto JTMB = JITTargetMachineBuilder(TI.getTriple()); - JTMB.addFeatures(TI.getTargetOpts().Features); - LLJITBuilder Builder; - Builder.setJITTargetMachineBuilder(JTMB); - Builder.setPrePlatformSetup( - [](LLJIT &J) { - // Try to enable debugging of JIT'd code (only works with JITLink for - // ELF and MachO). - consumeError(enableDebuggerSupport(J)); - return llvm::Error::success(); - }); - - if (auto JitOrErr = Builder.create()) + if (auto JitOrErr = JITBuilder.create()) Jit = std::move(*JitOrErr); else { Err = JitOrErr.takeError(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index dd0a210a0614..b4347209e14f 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -23,7 +23,9 @@ namespace llvm { class Error; namespace orc { +class JITTargetMachineBuilder; class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -44,8 +46,8 @@ class IncrementalExecutor { public: enum SymbolNameKind { IRName, LinkerName }; - IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err, - const clang::TargetInfo &TI); + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); ~IncrementalExecutor(); llvm::Error addModule(PartialTranslationUnit &PTU); @@ -56,6 +58,9 @@ public: getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const; llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + + static llvm::Expected> + createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); }; } // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 7fa52f2f15fc..cf31456b6950 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -372,15 +372,35 @@ Interpreter::Parse(llvm::StringRef Code) { return IncrParser->Parse(Code); } +static llvm::Expected +createJITTargetMachineBuilder(const std::string &TT) { + if (TT == llvm::sys::getProcessTriple()) + // This fails immediately if the target backend is not registered + return llvm::orc::JITTargetMachineBuilder::detectHost(); + + // If the target backend is not registered, LLJITBuilder::create() will fail + return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); +} + +llvm::Expected> +Interpreter::CreateJITBuilder(CompilerInstance &CI) { + auto JTMB = createJITTargetMachineBuilder(CI.getTargetOpts().Triple); + if (!JTMB) + return JTMB.takeError(); + return IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); +} + llvm::Error Interpreter::CreateExecutor() { - const clang::TargetInfo &TI = - getCompilerInstance()->getASTContext().getTargetInfo(); if (IncrExecutor) return llvm::make_error("Operation failed. " "Execution engine exists", std::error_code()); + llvm::Expected> JB = + CreateJITBuilder(*getCompilerInstance()); + if (!JB) + return JB.takeError(); llvm::Error Err = llvm::Error::success(); - auto Executor = std::make_unique(*TSCtx, Err, TI); + auto Executor = std::make_unique(*TSCtx, **JB, Err); if (!Err) IncrExecutor = std::move(Executor); diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index b7708616fd24..8bc429d9ec2d 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -18,14 +18,21 @@ #include "clang/Sema/Sema.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/Support/Error.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Threading.h" #include "llvm/Testing/Support/Error.h" #include "gmock/gmock.h" #include "gtest/gtest.h" + #include +#if defined(_AIX) +#define CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +#endif + using namespace clang; namespace { @@ -41,6 +48,10 @@ struct LLVMInitRAII { LLVMInitRAII() { llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmPrinter(); + LLVMInitializeARMTarget(); + LLVMInitializeARMTargetInfo(); + LLVMInitializeARMTargetMC(); + LLVMInitializeARMAsmPrinter(); } ~LLVMInitRAII() { llvm::llvm_shutdown(); } } LLVMInit; @@ -51,12 +62,30 @@ public: llvm::Error &Err) : Interpreter(std::move(CI), Err) {} - llvm::Error testCreateExecutor() { return Interpreter::CreateExecutor(); } + llvm::Error testCreateJITBuilderError() { + JB = nullptr; + return Interpreter::CreateExecutor(); + } + + llvm::Error testCreateExecutor() { + JB = std::make_unique(); + return Interpreter::CreateExecutor(); + } void resetExecutor() { Interpreter::ResetExecutor(); } + +private: + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JB) + return std::move(JB); + return llvm::make_error("TestError", std::error_code()); + } + + std::unique_ptr JB; }; -#ifdef _AIX +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { #else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { @@ -69,6 +98,8 @@ TEST(InterpreterExtensionsTest, ExecutorCreateReset) { llvm::Error ErrOut = llvm::Error::success(); TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut); cantFail(std::move(ErrOut)); + EXPECT_THAT_ERROR(Interp.testCreateJITBuilderError(), + llvm::FailedWithMessage("TestError")); cantFail(Interp.testCreateExecutor()); Interp.resetExecutor(); cantFail(Interp.testCreateExecutor()); @@ -126,4 +157,90 @@ TEST(InterpreterExtensionsTest, FindRuntimeInterface) { EXPECT_EQ(1U, Interp.RuntimeIBPtr->TransformerQueries); } +class CustomJBInterpreter : public Interpreter { + using CustomJITBuilderCreatorFunction = + std::function>()>; + CustomJITBuilderCreatorFunction JBCreator = nullptr; + +public: + CustomJBInterpreter(std::unique_ptr CI, llvm::Error &ErrOut) + : Interpreter(std::move(CI), ErrOut) {} + + ~CustomJBInterpreter() override { + // Skip cleanUp() because it would trigger LLJIT default dtors + Interpreter::ResetExecutor(); + } + + void setCustomJITBuilderCreator(CustomJITBuilderCreatorFunction Fn) { + JBCreator = std::move(Fn); + } + + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JBCreator) + return JBCreator(); + return Interpreter::CreateJITBuilder(CI); + } + + llvm::Error CreateExecutor() { return Interpreter::CreateExecutor(); } +}; + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) { +#else +TEST(InterpreterExtensionsTest, DefaultCrossJIT) { +#endif + IncrementalCompilerBuilder CB; + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + cantFail(Interp.CreateExecutor()); +} + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) { +#else +TEST(InterpreterExtensionsTest, CustomCrossJIT) { +#endif + std::string TargetTriple = "armv6-none-eabi"; + + IncrementalCompilerBuilder CB; + CB.SetTargetTriple(TargetTriple); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + + using namespace llvm::orc; + LLJIT *JIT = nullptr; + std::vector> Objs; + Interp.setCustomJITBuilderCreator([&]() { + auto JTMB = JITTargetMachineBuilder(llvm::Triple(TargetTriple)); + JTMB.setCPU("cortex-m0plus"); + auto JB = std::make_unique(); + JB->setJITTargetMachineBuilder(JTMB); + JB->setPlatformSetUp(setUpInactivePlatform); + JB->setNotifyCreatedCallback([&](LLJIT &J) { + ObjectLayer &ObjLayer = J.getObjLinkingLayer(); + auto *JITLinkObjLayer = llvm::dyn_cast(&ObjLayer); + JITLinkObjLayer->setReturnObjectBuffer( + [&Objs](std::unique_ptr MB) { + Objs.push_back(std::move(MB)); + }); + JIT = &J; + return llvm::Error::success(); + }); + return JB; + }); + + EXPECT_EQ(0U, Objs.size()); + cantFail(Interp.CreateExecutor()); + cantFail(Interp.ParseAndExecute("int a = 1;")); + ExecutorAddr Addr = cantFail(JIT->lookup("a")); + EXPECT_NE(0U, Addr.getValue()); + EXPECT_EQ(1U, Objs.size()); +} + } // end anonymous namespace -- GitLab From 75e528fdd9594ecb6fdb5d9e7bee1506f7e43be0 Mon Sep 17 00:00:00 2001 From: David Stuttard Date: Mon, 25 Mar 2024 09:01:46 +0000 Subject: [PATCH 064/879] [AMDGPU] Extend zero initialization of return values for TFE (#85759) buffer_load instructions that use TFE also need to zero initialize return values similar to how the image instructions currently work. Add support for this with standard zero init of all results + zero init of just TFE flag when enable-prt-strict-null subtarget feature is disabled. --- .../AMDGPU/AMDGPUInstructionSelector.cpp | 32 -- llvm/lib/Target/AMDGPU/BUFInstructions.td | 5 +- llvm/lib/Target/AMDGPU/MIMGInstructions.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 87 ++--- llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 6 + llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 + .../llvm.amdgcn.struct.buffer.load.format.ll | 1 + ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 1 + .../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 111 ++++-- .../llvm.amdgcn.struct.buffer.load.format.ll | 317 +++++++++++++++++- ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 280 ++++++++++++++++ 12 files changed, 741 insertions(+), 105 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 94cc1d90e0ca..e13c13913d4e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -2045,38 +2045,6 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic( if (BaseOpcode->HasD16) MIB.addImm(IsD16 ? -1 : 0); - if (IsTexFail) { - // An image load instruction with TFE/LWE only conditionally writes to its - // result registers. Initialize them to zero so that we always get well - // defined result values. - assert(VDataOut && !VDataIn); - Register Tied = MRI->cloneVirtualRegister(VDataOut); - Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) - .addImm(0); - auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); - if (STI.usePRTStrictNull()) { - // With enable-prt-strict-null enabled, initialize all result registers to - // zero. - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts) - RegSeq.addReg(Zero).addImm(Sub); - } else { - // With enable-prt-strict-null disabled, only initialize the extra TFE/LWE - // result register. - Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts.drop_back(1)) - RegSeq.addReg(Undef).addImm(Sub); - RegSeq.addReg(Zero).addImm(Parts.back()); - } - MIB.addReg(Tied, RegState::Implicit); - MIB->tieOperands(0, MIB->getNumOperands() - 1); - } - MI.eraseFromParent(); constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 4ae514ffcf78..273f92abf354 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -86,7 +86,7 @@ class BUF_Pseudo has_soffset = 1; bits<1> has_offset = 1; bits<1> has_slc = 1; - bits<1> tfe = ?; + bits<1> tfe = 0; bits<4> elements = 0; bits<1> has_sccb = 1; bits<1> sccb_value = 0; @@ -323,6 +323,7 @@ class MUBUF_Pseudo (MUBUFGetBaseOpcode.ret); let MUBUF = 1; let AsmMatchConverter = "cvtMubuf"; + let usesCustomInserter = 1; } class MUBUF_Real : @@ -3369,7 +3370,7 @@ def MUBUFInfoTable : GenericTable { let CppTypeName = "MUBUFInfo"; let Fields = [ "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", - "IsBufferInv" + "IsBufferInv", "tfe" ]; let PrimaryKey = ["Opcode"]; diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index 595ef39ce03e..23e8be0d5e45 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -210,6 +210,7 @@ class MIMG : MIMG_Base { let hasPostISelHook = 1; + let usesCustomInserter = 1; Instruction Opcode = !cast(NAME); MIMGBaseOpcode BaseOpcode; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d437f339a687..81a231f0cade 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -5410,6 +5410,11 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( return SplitBB; } default: + if (TII->isImage(MI) || TII->isMUBUF(MI)) { + if (!MI.mayStore()) + AddMemOpInit(MI); + return BB; + } return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); } } @@ -15034,60 +15039,67 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node, // result register that will be written in the case of a memory access failure. // The required code is also added to tie this init code to the result of the // img instruction. -void SITargetLowering::AddIMGInit(MachineInstr &MI) const { +void SITargetLowering::AddMemOpInit(MachineInstr &MI) const { const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo(); MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); MachineBasicBlock &MBB = *MI.getParent(); - MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); - MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); - MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - - if (!TFE && !LWE) // intersect_ray - return; + int DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + unsigned InitIdx = 0; - unsigned TFEVal = TFE ? TFE->getImm() : 0; - unsigned LWEVal = LWE ? LWE->getImm() : 0; - unsigned D16Val = D16 ? D16->getImm() : 0; + if (TII->isImage(MI)) { + MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); + MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); + MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - if (!TFEVal && !LWEVal) - return; + if (!TFE && !LWE) // intersect_ray + return; - // At least one of TFE or LWE are non-zero - // We have to insert a suitable initialization of the result value and - // tie this to the dest of the image instruction. + unsigned TFEVal = TFE ? TFE->getImm() : 0; + unsigned LWEVal = LWE ? LWE->getImm() : 0; + unsigned D16Val = D16 ? D16->getImm() : 0; - const DebugLoc &DL = MI.getDebugLoc(); + if (!TFEVal && !LWEVal) + return; - int DstIdx = - AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + // At least one of TFE or LWE are non-zero + // We have to insert a suitable initialization of the result value and + // tie this to the dest of the image instruction. - // Calculate which dword we have to initialize to 0. - MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); + // Calculate which dword we have to initialize to 0. + MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); - // check that dmask operand is found. - assert(MO_Dmask && "Expected dmask operand in instruction"); + // check that dmask operand is found. + assert(MO_Dmask && "Expected dmask operand in instruction"); - unsigned dmask = MO_Dmask->getImm(); - // Determine the number of active lanes taking into account the - // Gather4 special case - unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); + unsigned dmask = MO_Dmask->getImm(); + // Determine the number of active lanes taking into account the + // Gather4 special case + unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); - bool Packed = !Subtarget->hasUnpackedD16VMem(); + bool Packed = !Subtarget->hasUnpackedD16VMem(); - unsigned InitIdx = - D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; + InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; - // Abandon attempt if the dst size isn't large enough - // - this is in fact an error but this is picked up elsewhere and - // reported correctly. - uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; - if (DstSize < InitIdx) + // Abandon attempt if the dst size isn't large enough + // - this is in fact an error but this is picked up elsewhere and + // reported correctly. + uint32_t DstSize = + TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + if (DstSize < InitIdx) + return; + } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) { + InitIdx = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + } else { return; + } + + const DebugLoc &DL = MI.getDebugLoc(); // Create a register for the initialization value. - Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg()); unsigned NewDst = 0; // Final initialized value will be in here // If PRTStrictNull feature is enabled (the default) then initialize @@ -15185,11 +15197,8 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, return; } - if (TII->isImage(MI)) { - if (!MI.mayStore()) - AddIMGInit(MI); + if (TII->isImage(MI)) TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr); - } } static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 89da4428e3ab..9856a2923d38 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -466,7 +466,7 @@ public: SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override; - void AddIMGInit(MachineInstr &MI) const; + void AddMemOpInit(MachineInstr &MI) const; void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..a90dc32d396f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -318,6 +318,7 @@ struct MUBUFInfo { bool has_srsrc; bool has_soffset; bool IsBufferInv; + bool tfe; }; struct MTBUFInfo { @@ -466,6 +467,11 @@ bool getMUBUFIsBufferInv(unsigned Opc) { return Info ? Info->IsBufferInv : false; } +bool getMUBUFTfe(unsigned Opc) { + const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); + return Info ? Info->tfe : false; +} + bool getSMEMIsBuffer(unsigned Opc) { const SMInfo *Info = getSMEMOpcodeHelper(Opc); return Info ? Info->IsBuffer : false; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..f4f9a787100b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -525,6 +525,9 @@ bool getMUBUFHasSoffset(unsigned Opc); LLVM_READONLY bool getMUBUFIsBufferInv(unsigned Opc); +LLVM_READONLY +bool getMUBUFTfe(unsigned Opc); + LLVM_READONLY bool getSMEMIsBuffer(unsigned Opc); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll index 686b849ff58f..06bd45a45cce 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX12 %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; GFX8-LABEL: name: struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll index 9edc24554911..1e3f94a5e39c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; CHECK-LABEL: name: struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll index 1348315e72e7..7b1f55e7eeba 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll @@ -22,18 +22,36 @@ main_body: define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_both: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x00,0x00,0x60,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x05] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v9, v8 ; encoding: [0x08,0x03,0x12,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v10, v8 ; encoding: [0x08,0x03,0x14,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v11, v8 ; encoding: [0x08,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v8 ; encoding: [0x08,0x03,0x18,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:7], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x05,0x00,0x60,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x08,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_both: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x05] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0a,0x09] +; GFX12-NEXT: v_dual_mov_b32 v11, v8 :: v_dual_mov_b32 v12, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0c,0x0b] +; GFX12-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v7, v6, v5], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x07,0x06,0x05,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x08,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32i32.i32(i32 2, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0) @@ -63,18 +81,37 @@ main_body: define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, v3 ; encoding: [0x80,0x00,0x10,0xca,0x03,0x01,0x08,0x09] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v10, v9 ; encoding: [0x00,0x01,0x10,0xca,0x09,0x01,0x0a,0x05] +; GFX11-NEXT: v_mov_b32_e32 v11, v9 ; encoding: [0x09,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v9 ; encoding: [0x09,0x03,0x18,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v13, v9 ; encoding: [0x09,0x03,0x1a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:8], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x05,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x09,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2, v3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_mov_b32_e32 v9, 0 ; encoding: [0x80,0x02,0x12,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v6, v2 ; encoding: [0x03,0x01,0x10,0xca,0x02,0x01,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v0 ; encoding: [0x01,0x01,0x10,0xca,0x00,0x01,0x08,0x07] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x23,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v10, v9 :: v_dual_mov_b32 v11, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0a,0x0a] +; GFX12-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0c,0x0c] +; GFX12-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v8, v7, v6, v5], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x08,0x07,0x06,0x05] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x09,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32i32.i32(i32 8, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -155,18 +192,31 @@ main_body: define amdgpu_ps <4 x half> @load_2dmsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x03] +; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x05] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], v[3:5], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x03,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x06,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x03] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v6 ; encoding: [0x06,0x01,0x10,0xca,0x06,0x01,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x05,0x04,0x03,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x06,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -196,18 +246,31 @@ main_body: define amdgpu_ps <4 x half> @load_2darraymsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9c,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX11-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9d,0x01,0x62,0xf0,0x06,0x00,0x20,0x00,0x05,0x04,0x03,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x07,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX12-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7 ; encoding: [0x07,0x01,0x10,0xca,0x07,0x01,0x08,0x08] +; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x06,0x05,0x04,0x03] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x07,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll index 00be32b06de0..ba3d306cc0cf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-enable-prt-strict-null -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s ;RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s @@ -34,6 +35,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -75,6 +86,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -146,6 +164,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) { ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs_large: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -196,6 +233,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_12bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -235,6 +279,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_13bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -274,6 +327,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_16bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -313,6 +375,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_23bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -352,6 +423,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-SDAG-LABEL: buffer_load_voffset_large_24bit: ; GFX12-SDAG: ; %bb.0: ; %main_body ; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0x800000 :: v_dual_mov_b32 v0, 0 @@ -389,6 +469,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_idx: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], null idxen @@ -427,6 +513,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -466,6 +561,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs_imm: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -497,6 +601,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], null idxen offen @@ -529,6 +639,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i3 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both_reversed: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v2, v0 @@ -562,6 +679,13 @@ define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -595,6 +719,13 @@ define amdgpu_ps float @buffer_load_x_i32(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x_i32: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -629,6 +760,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_xy: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -644,7 +782,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 -; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX6-NEXT: v_mov_b32_e32 v7, 2 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 +; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s0, s2 @@ -658,7 +801,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 -; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX8PLUS-NEXT: v_mov_b32_e32 v7, 2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 +; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX8PLUS-NEXT: v_mov_b32_e32 v0, v6 @@ -667,22 +815,40 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 +; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v2, 2 +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4i32_tfe: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v2, 0 -; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe +; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 +; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX12-NEXT: v_mov_b32_e32 v0, v6 ; GFX12-NEXT: ; return to shader part epilog - %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0) + %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out %status = extractvalue { <4 x i32>, i32 } %load, 1 @@ -694,6 +860,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -708,6 +878,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -718,15 +892,32 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 ; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off @@ -744,6 +935,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -759,6 +953,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -769,15 +966,31 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -795,6 +1008,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -810,6 +1026,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -820,15 +1039,31 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -846,6 +1081,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -860,6 +1098,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -870,15 +1110,29 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -896,6 +1150,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -910,6 +1167,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -920,15 +1179,29 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -946,6 +1219,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -960,6 +1234,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -970,15 +1245,28 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off @@ -996,6 +1284,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -1010,6 +1299,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -1020,15 +1310,28 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll index b0bd4e428ef2..c5202b84fa1e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mattr=-enable-prt-strict-null -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) { ; GFX6-LABEL: buffer_load: @@ -31,6 +32,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp ; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0) %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1) @@ -62,6 +73,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 42, i32 0, i32 0) ret <4 x float> %data @@ -126,6 +144,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg) ; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog main_body: %d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0) %d.1 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 32764, i32 0) @@ -156,6 +193,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 0, i32 0) ret <4 x float> %data @@ -188,6 +232,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8188, i32 0, i32 0) ret <4 x float> %data @@ -220,6 +273,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 65532, i32 0, i32 0) ret <4 x float> %data @@ -252,6 +314,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8388604, i32 0, i32 0) ret <4 x float> %data @@ -284,6 +355,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 16777212, i32 0, i32 0) ret <4 x float> %data @@ -307,6 +387,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0, i32 0) ret <4 x float> %data @@ -339,6 +425,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -371,6 +466,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %ofs = add i32 %1, 60 %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %ofs, i32 0, i32 0) @@ -395,6 +499,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(ptr addrspace(8) inreg, i32, i32) ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 %2, i32 0, i32 0) ret <4 x float> %data @@ -421,6 +531,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(ptr addrspace(8) inreg, ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %2, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -447,6 +564,13 @@ define amdgpu_ps float @buffer_load_x(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret float %data @@ -473,6 +597,13 @@ define amdgpu_ps float @buffer_load_x_i32(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call i32 @llvm.amdgcn.struct.ptr.buffer.load.format.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %fdata = bitcast i32 %data to float @@ -500,6 +631,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret <2 x float> %data @@ -509,6 +647,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -523,6 +665,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -533,11 +679,25 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out @@ -550,6 +710,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -564,6 +728,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -574,11 +742,25 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x float>, i32 } %load, 0 store <4 x float> %data, ptr addrspace(1) %out @@ -591,6 +773,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -606,6 +791,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -616,11 +804,24 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x i32>, i32 } %load, 0 store <3 x i32> %data, ptr addrspace(1) %out @@ -633,6 +834,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -648,6 +852,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -658,11 +865,24 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x float>, i32 } %load, 0 store <3 x float> %data, ptr addrspace(1) %out @@ -675,6 +895,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -689,6 +912,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -699,11 +924,23 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x i32>, i32 } %load, 0 store <2 x i32> %data, ptr addrspace(1) %out @@ -716,6 +953,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -730,6 +970,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -740,11 +982,23 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x float>, i32 } %load, 0 store <2 x float> %data, ptr addrspace(1) %out @@ -757,6 +1011,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -771,6 +1026,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -781,11 +1037,22 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { i32, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { i32, i32 } %load, 0 store i32 %data, ptr addrspace(1) %out @@ -798,6 +1065,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -812,6 +1080,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -822,11 +1091,22 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { float, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { float, i32 } %load, 0 store float %data, ptr addrspace(1) %out -- GitLab From 99c40f6ba60ea6e5dbd3f015956c7d2f6e25e54c Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:13:42 +0100 Subject: [PATCH 065/879] [SPIR-V] Introduce a command line option to support compatibility with Khronos SPIRV Translator (#86101) SPIRV-LLVM-Translator project (https://github.com/KhronosGroup/SPIRV-LLVM-Translator) from Khronos Group is a tool and a library for bi-directional translation between SPIR-V and LLVM IR. In its backward translation from SPIR-V to LLVM IR SPIRV-LLVM-Translator isn't necessarily able to cover the same SPIR-V patterns/instructions set that SPIRV Backend produces, even if we target the same SPIR-V version in both SPIRV-LLVM-Translator and SPIRV Backend projects. To improve interoperability and ability to apply SPIRV Backend output in different products this PR introduces a notion of a mode of SPIR-V output that is compatible with a subset of SPIR-V supported by SPIRV-LLVM-Translator. This includes a new command line option that doesn't influence default behavior of SPIRV Backend and one test case that demonstrates how this command line option may be used to get a practical benefit of producing that one of two possible and similar output options that can be understood by SPIRV-LLVM-Translator. --- llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp | 8 +++++++- llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp index 38caa7c8ea0a..09a029a35a74 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -27,6 +27,11 @@ using namespace llvm; #define GET_SUBTARGETINFO_CTOR #include "SPIRVGenSubtargetInfo.inc" +static cl::opt + SPVTranslatorCompat("translator-compatibility-mode", + cl::desc("SPIR-V Translator compatibility mode"), + cl::Optional, cl::init(false)); + cl::list Extensions( "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore, cl::Hidden, @@ -157,8 +162,9 @@ bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const { } // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual. +// In SPIR-V Translator compatibility mode this feature is not available. bool SPIRVSubtarget::canDirectlyComparePointers() const { - return isAtLeastVer(SPIRVVersion, 14); + return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, 14); } void SPIRVSubtarget::initAvailableExtensions() { diff --git a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll index 641e2bf0649c..31cd8bd45929 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll @@ -1,7 +1,13 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - | FileCheck %s --check-prefix=CHECK-COMPAT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[EQ:%.*]] "test_eq" ; CHECK-DAG: OpName [[NE:%.*]] "test_ne" +; CHECK-COMPAT-DAG: OpName [[EQ:%.*]] "test_eq" +; CHECK-COMPAT-DAG: OpName [[NE:%.*]] "test_ne" ; CHECK-DAG: OpName [[ULT:%.*]] "test_ult" ; CHECK-DAG: OpName [[SLT:%.*]] "test_slt" ; CHECK-DAG: OpName [[ULE:%.*]] "test_ule" @@ -19,6 +25,9 @@ ; CHECK-NEXT: [[R:%.*]] = OpPtrEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[EQ]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_eq(i16* %a, i16* %b) { %r = icmp eq i16* %a, %b ret i1 %r @@ -31,6 +40,9 @@ define i1 @test_eq(i16* %a, i16* %b) { ; CHECK-NEXT: [[R:%.*]] = OpPtrNotEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[NE]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrNotEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_ne(i16* %a, i16* %b) { %r = icmp ne i16* %a, %b ret i1 %r -- GitLab From 1d250d9099a9ba8b53add7eb7db6827e8fc0c8fd Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:08 +0100 Subject: [PATCH 066/879] [SPIR-V] Improve type inference in SPIR-V Backend for opaque pointers (#86283) This PR improves type inference in SPIR-V Backend for opaque pointers, accounting or a case when there is a chain of function calls that allows to deduce formal parameter types from actual arguments. The attached test demonstrates the case. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 123 +++++++++++------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 12 +- .../pointers/type-deduce-by-call-chain.ll | 52 ++++++++ 4 files changed, 138 insertions(+), 50 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 458af9229ed7..5828db6669ff 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -92,6 +92,9 @@ class SPIRVEmitIntrinsics void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); void processParamTypes(Function *F, IRBuilder<> &B); + Type *deduceFunParamType(Function *F, unsigned OpIdx); + Type *deduceFunParamType(Function *F, unsigned OpIdx, + std::unordered_set &FVisited); public: static char ID; @@ -169,6 +172,10 @@ static inline void reportFatalOnTokenType(const Instruction *I) { static Type *deduceElementTypeHelper(Value *I, std::unordered_set &Visited, DenseMap &DeducedElTys) { + // allow to pass nullptr as an argument + if (!I) + return nullptr; + // maybe already known auto It = DeducedElTys.find(I); if (It != DeducedElTys.end()) @@ -182,15 +189,20 @@ static Type *deduceElementTypeHelper(Value *I, // fallback value in case when we fail to deduce a type Type *Ty = nullptr; // look for known basic patterns of type inference - if (auto *Ref = dyn_cast(I)) + if (auto *Ref = dyn_cast(I)) { Ty = Ref->getAllocatedType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getResultElementType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getValueType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, DeducedElTys); + } else if (auto *Ref = dyn_cast(I)) { + if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); + isPointerTy(Src) && isPointerTy(Dest)) + Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, DeducedElTys); + } // remember the found relationship if (Ty) @@ -795,61 +807,80 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, } } -void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { - DenseMap Args; - unsigned i = 0; - for (Argument &Arg : F->args()) { - if (isUntypedPointerTy(Arg.getType()) && - DeducedElTys.find(&Arg) == DeducedElTys.end() && - !HasPointeeTypeAttr(&Arg)) - Args[i] = &Arg; - i++; - } - if (Args.size() == 0) - return; +Type *SPIRVEmitIntrinsics::deduceFunParamType(Function *F, unsigned OpIdx) { + std::unordered_set FVisited; + return deduceFunParamType(F, OpIdx, FVisited); +} + +Type *SPIRVEmitIntrinsics::deduceFunParamType( + Function *F, unsigned OpIdx, std::unordered_set &FVisited) { + // maybe a cycle + if (FVisited.find(F) != FVisited.end()) + return nullptr; + FVisited.insert(F); - // Args contains opaque pointers without element type definition - B.SetInsertPointPastAllocas(F); std::unordered_set Visited; + SmallVector> Lookup; + // search in function's call sites for (User *U : F->users()) { CallInst *CI = dyn_cast(U); - if (!CI) + if (!CI || OpIdx >= CI->arg_size()) continue; - for (unsigned OpIdx = 0; OpIdx < CI->arg_size() && Args.size() > 0; - OpIdx++) { - auto It = Args.find(OpIdx); - Argument *Arg = It == Args.end() ? nullptr : It->second; - if (!Arg) - continue; - Value *OpArg = CI->getArgOperand(OpIdx); - if (!isPointerTy(OpArg->getType())) + Value *OpArg = CI->getArgOperand(OpIdx); + if (!isPointerTy(OpArg->getType())) + continue; + // maybe we already know operand's element type + if (auto It = DeducedElTys.find(OpArg); It != DeducedElTys.end()) + return It->second; + // search in actual parameter's users + for (User *OpU : OpArg->users()) { + Instruction *Inst = dyn_cast(OpU); + if (!Inst || Inst == CI) continue; - // maybe we already know the operand's element type - auto DeducedIt = DeducedElTys.find(OpArg); - Type *ElemTy = - DeducedIt == DeducedElTys.end() ? nullptr : DeducedIt->second; - if (!ElemTy) { - for (User *OpU : OpArg->users()) { - if (Instruction *Inst = dyn_cast(OpU)) { - Visited.clear(); - ElemTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys); - if (ElemTy) - break; - } - } + Visited.clear(); + if (Type *Ty = deduceElementTypeHelper(Inst, Visited, DeducedElTys)) + return Ty; + } + // check if it's a formal parameter of the outer function + if (!CI->getParent() || !CI->getParent()->getParent()) + continue; + Function *OuterF = CI->getParent()->getParent(); + if (FVisited.find(OuterF) != FVisited.end()) + continue; + for (unsigned i = 0; i < OuterF->arg_size(); ++i) { + if (OuterF->getArg(i) == OpArg) { + Lookup.push_back(std::make_pair(OuterF, i)); + break; } - if (ElemTy) { - unsigned AddressSpace = getPointerAddressSpace(Arg->getType()); + } + } + + // search in function parameters + for (auto &Pair : Lookup) { + if (Type *Ty = deduceFunParamType(Pair.first, Pair.second, FVisited)) + return Ty; + } + + return nullptr; +} + +void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { + B.SetInsertPointPastAllocas(F); + DenseMap Args; + for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { + Argument *Arg = F->getArg(OpIdx); + if (isUntypedPointerTy(Arg->getType()) && + DeducedElTys.find(Arg) == DeducedElTys.end() && + !HasPointeeTypeAttr(Arg)) { + if (Type *ElemTy = deduceFunParamType(F, OpIdx)) { CallInst *AssignPtrTyCI = buildIntrWithMD( Intrinsic::spv_assign_ptr_type, {Arg->getType()}, - Constant::getNullValue(ElemTy), Arg, {B.getInt32(AddressSpace)}, B); + Constant::getNullValue(ElemTy), Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); DeducedElTys[AssignPtrTyCI] = ElemTy; DeducedElTys[Arg] = ElemTy; - Args.erase(It); } } - if (Args.size() == 0) - break; } } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 42f8397a3023..ee52163a5d12 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -479,6 +479,7 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( GVar = M->getGlobalVariable(Name); if (GVar == nullptr) { const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type. + // Module takes ownership of the global var. GVar = new GlobalVariable(*M, const_cast(Ty), false, GlobalValue::ExternalLinkage, nullptr, Twine(Name)); diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 5bb8f6084f96..39228e2196b3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -499,6 +499,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); Register GV = I.getOperand(1).getReg(); MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); + (void)II; assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || (*II).getOpcode() == TargetOpcode::COPY || (*II).getOpcode() == SPIRV::OpVariable) && @@ -771,10 +772,13 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); // TODO: check if we have such GV, add init, use buildGlobalVariable. - Type *LLVMArrTy = ArrayType::get( - IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); - GlobalVariable *GV = - new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); + Function &CurFunction = GR.CurMF->getFunction(); + Type *LLVMArrTy = + ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); + // Module takes ownership of the global var. + GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, + true, GlobalValue::InternalLinkage, + Constant::getNullValue(LLVMArrTy)); Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(GV, GR.CurMF, VarReg); diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll new file mode 100644 index 000000000000..703f1e22a032 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll @@ -0,0 +1,52 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[ArgCum:.*]] "_arg_cum" +; CHECK-SPIRV-DAG: OpName %[[FunTest:.*]] "test" +; CHECK-SPIRV-DAG: OpName %[[Addr:.*]] "addr" +; CHECK-SPIRV-DAG: OpName %[[StubObj:.*]] "stub_object" +; CHECK-SPIRV-DAG: OpName %[[MemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooStub:.*]] "foo_stub" +; CHECK-SPIRV-DAG: OpName %[[FooObj:.*]] "foo_object" +; CHECK-SPIRV-DAG: OpName %[[FooMemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooFunc:.*]] "foo" +; CHECK-SPIRV-DAG: %[[TyLong:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[TyVoid:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[TyPtrLong:.*]] = OpTypePointer CrossWorkgroup %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunPtrLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrLong]] +; CHECK-SPIRV-DAG: %[[TyGenPtrLong:.*]] = OpTypePointer Generic %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunGenPtrLongLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyGenPtrLong]] %[[TyLong]] +; CHECK-SPIRV-DAG: %[[Const3:.*]] = OpConstant %[[TyLong]] 3 +; CHECK-SPIRV: %[[FunTest]] = OpFunction %[[TyVoid]] None %[[TyFunPtrLong]] +; CHECK-SPIRV: %[[ArgCum]] = OpFunctionParameter %[[TyPtrLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooFunc]] %[[Addr]] %[[Const3]] +; CHECK-SPIRV: %[[FooStub]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[StubObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[MemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: %[[FooFunc]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[FooObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[FooMemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooStub]] %[[FooObj]] %[[FooMemOrder]] + +define spir_kernel void @test(ptr addrspace(1) noundef align 4 %_arg_cum) { +entry: + %lptr = getelementptr inbounds i32, ptr addrspace(1) %_arg_cum, i64 1 + %addr = addrspacecast ptr addrspace(1) %lptr to ptr addrspace(4) + %object = bitcast ptr addrspace(4) %addr to ptr addrspace(4) + call spir_func void @foo(ptr addrspace(4) %object, i32 3) + ret void +} + +define void @foo_stub(ptr addrspace(4) noundef %stub_object, i32 noundef %mem_order) { +entry: + %object.addr = alloca ptr addrspace(4) + %object.addr.ascast = addrspacecast ptr %object.addr to ptr addrspace(4) + store ptr addrspace(4) %stub_object, ptr addrspace(4) %object.addr.ascast + ret void +} + +define void @foo(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) { + tail call void @foo_stub(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) + ret void +} + -- GitLab From b0d03ccc0855f2bff39160f25fcde06aae07cace Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:46 +0100 Subject: [PATCH 067/879] [SPIR-V] Fix illegal OpConstantComposite instruction with non-const constituents in SPIR-V Backend (#86352) This PR fixes illegal use of OpConstantComposite with non-constant constituents. The test attached to the PR is able now to satisfy `spirv-val` check. Before the fix SPIR-V Backend produced for the attached test case a pattern like ``` %a = OpVariable %_ptr_CrossWorkgroup_uint CrossWorkgroup %uint_123 %11 = OpConstantComposite %_struct_6 %a %a ``` so that `spirv-val` complained with ``` error: line 25: OpConstantComposite Constituent '10[%a]' is not a constant or undef. %11 = OpConstantComposite %_struct_6 %a %a ``` --- .../Target/SPIRV/SPIRVDuplicatesTracker.cpp | 1 + .../lib/Target/SPIRV/SPIRVDuplicatesTracker.h | 9 ++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 8 ++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 90 +++++++++++++++---- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 1 + .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 1 + .../SPIRV/pointers/struct-opaque-pointers.ll | 2 +- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp index d82fb2df4539..7c32bb1968ef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp @@ -39,6 +39,7 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph( prebuildReg2Entry(GT, Reg2Entry); prebuildReg2Entry(FT, Reg2Entry); prebuildReg2Entry(AT, Reg2Entry); + prebuildReg2Entry(MT, Reg2Entry); prebuildReg2Entry(ST, Reg2Entry); for (auto &Op2E : Reg2Entry) { diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h index 96cc621791e9..2ec3fb35ca04 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h @@ -262,6 +262,7 @@ class SPIRVGeneralDuplicatesTracker { SPIRVDuplicatesTracker GT; SPIRVDuplicatesTracker FT; SPIRVDuplicatesTracker AT; + SPIRVDuplicatesTracker MT; SPIRVDuplicatesTracker ST; // NOTE: using MOs instead of regs to get rid of MF dependency to be able @@ -306,6 +307,10 @@ public: AT.add(Arg, MF, R); } + void add(const MachineInstr *MI, const MachineFunction *MF, Register R) { + MT.add(MI, MF, R); + } + void add(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF, Register R) { ST.add(TD, MF, R); @@ -337,6 +342,10 @@ public: return AT.find(const_cast(Arg), MF); } + Register find(const MachineInstr *MI, const MachineFunction *MF) { + return MT.find(const_cast(MI), MF); + } + Register find(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF) { return ST.find(TD, MF); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index ee52163a5d12..db66ed4f0e0f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -123,6 +123,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVType *ElemType, MachineIRBuilder &MIRBuilder) { auto EleOpc = ElemType->getOpcode(); + (void)EleOpc; assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat || EleOpc == SPIRV::OpTypeBool) && "Invalid vector element type"); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index da480b22a525..ed0f90ff89ce 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -94,6 +94,14 @@ public: DT.add(Arg, MF, R); } + void add(const MachineInstr *MI, MachineFunction *MF, Register R) { + DT.add(MI, MF, R); + } + + Register find(const MachineInstr *MI, MachineFunction *MF) { + return DT.find(MI, MF); + } + Register find(const Constant *C, MachineFunction *MF) { return DT.find(C, MF); } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 39228e2196b3..505b19a4d66e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -231,6 +231,9 @@ private: Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const; + + bool wrapIntoSpecConstantOp(MachineInstr &I, + SmallVector &CompositeArgs) const; }; } // end anonymous namespace @@ -1249,6 +1252,24 @@ static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, return N; } +// Return true if the type represents a constant register +static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || + OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; +} + +// Return true if the virtual register represents a constant +static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { + if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) + return isConstReg(MRI, OpDef); + return false; +} + bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1266,16 +1287,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, // check if we may construct a constant vector Register OpReg = I.getOperand(OpIdx).getReg(); - bool IsConst = false; - if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) { - if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && - OpDef->getOperand(1).isReg()) { - if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) - OpDef = RefDef; - } - IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT || - OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; - } + bool IsConst = isConstReg(MRI, OpReg); if (!IsConst && N < 2) report_fatal_error( @@ -1628,6 +1640,48 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg, return Res.constrainAllUses(TII, TRI, RBI); } +// Maybe wrap a value into OpSpecConstantOp +bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( + MachineInstr &I, SmallVector &CompositeArgs) const { + bool Result = true; + unsigned Lim = I.getNumExplicitOperands(); + for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { + Register OpReg = I.getOperand(i).getReg(); + SPIRVType *OpDefine = MRI->getVRegDef(OpReg); + SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); + if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || + OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { + // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed + // by selectAddrSpaceCast() + CompositeArgs.push_back(OpReg); + continue; + } + MachineFunction *MF = I.getMF(); + Register WrapReg = GR.find(OpDefine, MF); + if (WrapReg.isValid()) { + CompositeArgs.push_back(WrapReg); + continue; + } + // Create a new register for the wrapper + WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); + GR.add(OpDefine, MF, WrapReg); + CompositeArgs.push_back(WrapReg); + // Decorate the wrapper register and generate a new instruction + MRI->setType(WrapReg, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); + MachineBasicBlock &BB = *I.getParent(); + Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) + .addDef(WrapReg) + .addUse(GR.getSPIRVTypeID(OpType)) + .addImm(static_cast(SPIRV::Opcode::Bitcast)) + .addUse(OpReg) + .constrainAllUses(TII, TRI, RBI); + if (!Result) + break; + } + return Result; +} + bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1666,17 +1720,21 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_const_composite: { // If no values are attached, the composite is null constant. bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); - unsigned Opcode = - IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; + // Select a proper instruction. + unsigned Opcode = SPIRV::OpConstantNull; + SmallVector CompositeArgs; + if (!IsNull) { + Opcode = SPIRV::OpConstantComposite; + if (!wrapIntoSpecConstantOp(I, CompositeArgs)) + return false; + } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // skip type MD node we already used when generated assign.type for this if (!IsNull) { - for (unsigned i = I.getNumExplicitDefs() + 1; - i < I.getNumExplicitOperands(); ++i) { - MIB.addUse(I.getOperand(i).getReg()); - } + for (Register OpReg : CompositeArgs) + MIB.addUse(OpReg); } return MIB.constrainAllUses(TII, TRI, RBI); } diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index d547f91ba4a5..1f0d8d8cd43a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -543,6 +543,7 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR, Register Dst = ICMP->getOperand(0).getReg(); MachineOperand &PredOp = ICMP->getOperand(1); const auto CC = static_cast(PredOp.getPredicate()); + (void)CC; assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) && MRI.hasOneUse(Dst) && MRI.hasOneDef(CompareReg)); uint64_t Value = getIConstVal(ICMP->getOperand(3).getReg(), &MRI); diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 8dbbd9049844..ff102e318469 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1611,3 +1611,4 @@ multiclass OpcodeOperand value> { // TODO: implement other mnemonics. defm InBoundsPtrAccessChain : OpcodeOperand<70>; defm PtrCastToGeneric : OpcodeOperand<121>; +defm Bitcast : OpcodeOperand<124>; diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll index d426fc4dfd4e..ce3ab8895a59 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]] -- GitLab From cbbbab349e9c412729c3969008cdcb677cc55790 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 12:41:32 +0000 Subject: [PATCH 068/879] [RemoveDIs] Enable direct-to-bitcode writing by default Follow on from #83251. This patch simply enables the behaviour by default in order to provide an easily revertible capstone. --- llvm/lib/IR/BasicBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index f088c7a2cc4e..ae99267f5ba8 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -39,7 +39,7 @@ cl::opt bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, - cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true)); DbgMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && -- GitLab From 3cb024198f6a028089779c0a3cb4d8e753e87c73 Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Mon, 25 Mar 2024 17:17:36 +0800 Subject: [PATCH 069/879] [RISCV][CostModel] Estimate cost of llvm.vector.reduce.fmaximum/fminimum (#80697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ‘llvm.vector.reduce.fmaximum/fminimum.*’ intrinsics propagate NaNs if any element of the vector is a NaN. Following #79402, the patch adds the cost for NaN check (vmfne + vcpop) --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 +++++++++ .../CostModel/RISCV/reduce-fmaximum.ll | 91 +++++++++++++------ .../CostModel/RISCV/reduce-fminimum.ll | 52 +++++------ 3 files changed, 136 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 8f46fdc2f7ca..f75b3d3caa62 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1001,6 +1001,51 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind); } + if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) { + SmallVector Opcodes; + InstructionCost ExtraCost = 0; + switch (IID) { + case Intrinsic::maximum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DstTy->getScalarSizeInBits(); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + + case Intrinsic::minimum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + } + return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind); + } + // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll index 1618c3833a97..f91f13b2d9ec 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll @@ -6,23 +6,37 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) @@ -32,6 +46,13 @@ define float @reduce_fmaximum_f32(float %arg) { %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ret float undef } declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>) @@ -44,21 +65,33 @@ declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) @@ -67,6 +100,12 @@ define double @reduce_fmaximum_f64(double %arg) { %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ret double undef } declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll index 35b18645b1f2..86b84025ad54 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll @@ -6,23 +6,23 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) @@ -44,21 +44,21 @@ declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -- GitLab From e550022b68fc75d32e44faa59ed8f02036cd8f26 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:26:27 +0000 Subject: [PATCH 070/879] [RemoveDIs] Load into new debug info format by default in llvm-dis (#86276) Directly load all bitcode into the new debug info format in llvm-dis. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-dis/llvm-dis.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp index 8e443318dd7d..49154dc46c57 100644 --- a/llvm/tools/llvm-dis/llvm-dis.cpp +++ b/llvm/tools/llvm-dis/llvm-dis.cpp @@ -82,6 +82,8 @@ static cl::opt PrintThinLTOIndexOnly( extern cl::opt WriteNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + namespace { static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) { @@ -169,6 +171,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&DisCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + LLVMContext Context; Context.setDiagnosticHandler( std::make_unique(argv[0])); -- GitLab From 2ef612050844355906e4b67d892a00bbb58c41d6 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:28:01 +0000 Subject: [PATCH 071/879] [RemoveDIs] Do not load into new debug info format from bitcode by default (#86268) This is NFC right now, as the global default behaviour is also "do not load into the new debug info format by default", but we want to change that soon. Additionally unconditionally convert from the new debug info format into if we've loaded into it (e.g., if the bitcode file loaded was already in the new format). The latter change is needed because verify-uselistorder doesn't yet understand DbgRecords (it doesn't know how to map them). The former change is needed because if we load from an old debug format bitcode file but load directly into the new format _and then convert back to the old mode after_, the use-lists of the debug intrinsic functions (the functions' global value uses) change. --- .../verify-uselistorder/verify-uselistorder.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index d929ae09958a..8b299c394e4e 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,6 +68,8 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); +extern cl::opt LoadBitcodeIntoNewDbgInforFormat; + namespace { struct TempFile { @@ -169,8 +171,7 @@ std::unique_ptr TempFile::readBitcode(LLVMContext &Context) const { // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!ModuleOr.get()->IsNewDbgInfoFormat && - "Unexpectedly in new debug info mode"); + ModuleOr.get()->setIsNewDbgInfoFormat(false); return std::move(ModuleOr.get()); } @@ -182,7 +183,7 @@ std::unique_ptr TempFile::readAssembly(LLVMContext &Context) const { Err.print("verify-uselistorder", errs()); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); return M; } @@ -544,6 +545,10 @@ int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "llvm tool to verify use-list order\n"); + // Do not load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + LLVMContext Context; SMDiagnostic Err; @@ -551,7 +556,7 @@ int main(int argc, char **argv) { std::unique_ptr M = parseIRFile(InputFilename, Err, Context); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); if (!M.get()) { Err.print(argv[0], errs()); -- GitLab From 8263a883342d9925a4a1fd9752efc8deda5840fc Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:29:19 +0000 Subject: [PATCH 072/879] [RemoveDIs] Load into new debug info format by default in llvm-link (#86274) Directly load all bitcode into the new debug info format in llvm-link. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-link/llvm-link.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp index 9e7f2c3ebac4..9049cb5e8580 100644 --- a/llvm/tools/llvm-link/llvm-link.cpp +++ b/llvm/tools/llvm-link/llvm-link.cpp @@ -136,6 +136,8 @@ static cl::opt TryUseNewDbgInfoFormat( extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + static ExitOnError ExitOnErr; // Read the specified bitcode file in and return it. This routine searches the @@ -480,6 +482,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&LinkCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. if (TryUseNewDbgInfoFormat) { -- GitLab From 772e316457ef94759804d9f4da0af70d8d2ca4d4 Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Mon, 25 Mar 2024 09:43:41 +0000 Subject: [PATCH 073/879] [FMV] Allow multi versioning without default declaration. (#85454) This was a limitation which has now been lifted. Please read the thread below for more details: https://github.com/llvm/llvm-project/pull/84405#discussion_r1525583647 Basically it allows to separate versioned implementations across different TUs without having to share private header files which contain the default declaration. The ACLE spec has been updated accordingly to make this explicit: "Each version declaration should be visible at the translation unit in which the corresponding function version resides." https://github.com/ARM-software/acle/pull/310 If a resolver is required (because there is a caller in the TU), then a default declaration is implicitly generated. --- clang/lib/CodeGen/CodeGenModule.cpp | 131 ++++--- clang/lib/Sema/SemaDecl.cpp | 6 +- clang/lib/Sema/SemaOverload.cpp | 44 ++- clang/test/CodeGen/attr-target-version.c | 368 ++++++++++++------ clang/test/CodeGenCXX/attr-target-version.cpp | 62 +-- clang/test/Sema/aarch64-sme-func-attrs.c | 8 +- clang/test/Sema/attr-target-version.c | 8 +- clang/test/SemaCXX/attr-target-version.cpp | 11 +- 8 files changed, 403 insertions(+), 235 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index cb153066b28d..ac81df8cf7ad 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3711,7 +3711,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + !FD->isTargetVersionMultiVersion()) return; StringRef MangledName = getMangledName(GD); @@ -4092,6 +4093,23 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM, return llvm::GlobalValue::WeakODRLinkage; } +static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) { + DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl(); + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + StorageClass SC = FD->getStorageClass(); + DeclarationName Name = FD->getNameInfo().getName(); + + FunctionDecl *NewDecl = + FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(), + FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC); + + NewDecl->setIsMultiVersion(); + NewDecl->addAttr(TargetVersionAttr::CreateImplicit( + NewDecl->getASTContext(), "default", NewDecl->getSourceRange())); + + return NewDecl; +} + void CodeGenModule::emitMultiVersionFunctions() { std::vector MVFuncsToEmit; MultiVersionFuncs.swap(MVFuncsToEmit); @@ -4099,70 +4117,54 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - bool EmitResolver = !FD->isTargetVersionMultiVersion(); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + return cast(Func); + }; + + bool HasDefaultDecl = !FD->isTargetVersionMultiVersion(); + bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion(); SmallVector Options; if (FD->isTargetMultiVersion()) { getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr(); - llvm::SmallVector Feats; + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector Feats; + llvm::Function *Func = createFunction(CurFD); + + if (const auto *TA = CurFD->getAttr()) { TA->getAddedFeatures(Feats); - Options.emplace_back(cast(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr(); - if (CurFD->isUsed() || (TVA->isDefaultVersion() && - CurFD->doesThisDeclarationHaveABody())) - EmitResolver = true; - llvm::SmallVector Feats; + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr()) { + bool HasDefaultDef = TVA->isDefaultVersion() && + CurFD->doesThisDeclarationHaveABody(); + HasDefaultDecl |= TVA->isDefaultVersion(); + ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef); TVA->getFeatures(Feats); - Options.emplace_back(cast(Func), - /*Architecture*/ "", Feats); - } + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else + llvm_unreachable("unexpected MultiVersionKind"); }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) + } else if (const auto *TC = FD->getAttr()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } + llvm::Function *Func = createFunction(FD, I); + StringRef Version = TC->getFeatureStr(I); StringRef Architecture; llvm::SmallVector Feature; @@ -4180,16 +4182,23 @@ void CodeGenModule::emitMultiVersionFunctions() { Feature.push_back(Version); } - Options.emplace_back(cast(Func), Architecture, Feature); + Options.emplace_back(Func, Architecture, Feature); } } else { assert(0 && "Expected a target or target_clones multiversion function"); continue; } - if (!EmitResolver) + if (!ShouldEmitResolver) continue; + if (!HasDefaultDecl) { + FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD); + llvm::Function *Func = createFunction(NewFD); + llvm::SmallVector Feats; + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); @@ -4480,7 +4489,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) + if (FD->isTargetVersionMultiVersion() && !FD->isUsed()) + AddDeferredMultiVersionResolverToEmit(GD); + else if (!IsForDefinition) return GetOrCreateMultiVersionResolver(GD); } } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index aa754d47a0c4..73ea155053d7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -11441,9 +11441,9 @@ static bool CheckMultiVersionFirstFunction(Sema &S, FunctionDecl *FD) { "Function lacks multiversion attribute"); const auto *TA = FD->getAttr(); const auto *TVA = FD->getAttr(); - // Target and target_version only causes MV if it is default, otherwise this - // is a normal function. - if ((TA && !TA->isDefaultVersion()) || (TVA && !TVA->isDefaultVersion())) + // The target attribute only causes MV if this declaration is the default, + // otherwise it is treated as a normal function. + if (TA && !TA->isDefaultVersion()) return false; if ((TA || TVA) && CheckMultiVersionValue(S, FD)) { diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index f6bd85bdc646..51450e486eae 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -6865,6 +6865,32 @@ static bool IsAcceptableNonMemberOperatorCandidate(ASTContext &Context, return false; } +static bool isNonViableMultiVersionOverload(FunctionDecl *FD) { + if (FD->isTargetMultiVersionDefault()) + return false; + + if (!FD->getASTContext().getTargetInfo().getTriple().isAArch64()) + return FD->isTargetMultiVersion(); + + if (!FD->isMultiVersion()) + return false; + + // Among multiple target versions consider either the default, + // or the first non-default in the absence of default version. + unsigned SeenAt = 0; + unsigned I = 0; + bool HasDefault = false; + FD->getASTContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + if (FD == CurFD) + SeenAt = I; + else if (CurFD->isTargetMultiVersionDefault()) + HasDefault = true; + ++I; + }); + return HasDefault || SeenAt != 0; +} + /// AddOverloadCandidate - Adds the given function to the set of /// candidate functions, using the given function call arguments. If /// @p SuppressUserConversions, then don't allow user-defined @@ -6970,11 +6996,7 @@ void Sema::AddOverloadCandidate( } } - if (Function->isMultiVersion() && - ((Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()) || - (Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Function)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; return; @@ -7637,11 +7659,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, return; } - if (Method->isMultiVersion() && - ((Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()) || - (Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Method)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } @@ -8127,11 +8145,7 @@ void Sema::AddConversionCandidate( return; } - if (Conversion->isMultiVersion() && - ((Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()) || - (Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Conversion)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index 25129605e76c..dd4cbbf5a898 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -109,21 +109,47 @@ int unused_with_implicit_default_def(void) { return 1; } int unused_with_implicit_forward_default_def(void) { return 0; } __attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; } -// This should generate a normal function. +// This should generate a target version despite the default not being declared. __attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; } +// These shouldn't generate anything. +int unused_version_declarations(void); +__attribute__((target_version("jscvt"))) int unused_version_declarations(void); +__attribute__((target_version("rdma"))) int unused_version_declarations(void); + +// These should generate the default (mangled) version and the resolver. +int default_def_with_version_decls(void) { return 0; } +__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void); +__attribute__((target_version("rdma"))) int default_def_with_version_decls(void); + +// The following is guarded because in NOFMV we get errors for calling undeclared functions. +#ifdef __HAVE_FUNCTION_MULTI_VERSIONING +// This should generate a default declaration, two target versions and the resolver. +__attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; } +__attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; } + +// This should generate a default declaration and the resolver. +__attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void); +__attribute__((target_version("rdma"))) int used_decl_without_default_decl(void); + +int caller(void) { return used_def_without_default_decl() + used_decl_without_default_decl(); } +#endif + //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @fmv.ifunc = weak_odr alias i32 (), ptr @fmv // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e +// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline -// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def // CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def // CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def +// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls +// CHECK: @used_def_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_def_without_default_decl +// CHECK: @used_decl_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_decl_without_default_decl // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver @@ -131,97 +157,121 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver +// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver +// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver +// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver //. // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 -// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-SAME: () #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata -// CHECK-SAME: () #[[ATTR2:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag -// CHECK-SAME: () #[[ATTR3:[0-9]+]] { +// CHECK-SAME: () #[[ATTR4:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp -// CHECK-SAME: () #[[ATTR4:[0-9]+]] { +// CHECK-SAME: () #[[ATTR5:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v -// CHECK-SAME: () #[[ATTR5:[0-9]+]] { +// CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti -// CHECK-SAME: () #[[ATTR6:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 -// CHECK-SAME: () #[[ATTR7:[0-9]+]] { +// CHECK-SAME: () #[[ATTR8:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb -// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-SAME: () #[[ATTR10:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh -// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd -// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -371,35 +421,49 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d.default +// CHECK-SAME: () #[[ATTR11]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -587,7 +651,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -595,7 +659,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -606,7 +670,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -623,228 +687,268 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR12:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 +// CHECK-SAME: () #[[ATTR12]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm +// CHECK-SAME: () #[[ATTR18:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 -// CHECK-SAME: () #[[ATTR10]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm +// CHECK-SAME: () #[[ATTR18]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_without_default -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@caller +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() +// CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() +// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] +// CHECK-NEXT: ret i32 [[ADD]] +// +// +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_def_without_default_decl.default +// +// +// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl.default // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR19:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR20:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR13]] { +// CHECK-SAME: () #[[ATTR15]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd -// CHECK-SAME: () #[[ATTR3]] { +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 14 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR34:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-SAME: () #[[ATTR35:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR9]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 -// -// // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -887,6 +991,28 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def.default // // +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @default_def_with_version_decls.default +// +// // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { @@ -995,40 +1121,50 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NOFMV-NEXT: entry: // CHECK-NOFMV-NEXT: ret i32 0 // +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 0 +// //. -// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index e06121d1a719..8b7273fe3bb5 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -35,7 +35,7 @@ struct MyClass { int unused_with_implicit_forward_default_def(void); int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void); - // This should generate a normal function. + // This should generate a target version despite the default not being declared. int __attribute__((target_version("rdm"))) unused_without_default(void); }; @@ -75,6 +75,13 @@ int bar() { // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver //. +// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: ret i32 1 +// +// // CHECK-LABEL: @_Z3fooi.default( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 @@ -82,6 +89,11 @@ int bar() { // CHECK-NEXT: ret i32 2 // // +// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 3 +// +// // CHECK-LABEL: @_Z3foov.default( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -189,6 +201,14 @@ int bar() { // CHECK-NEXT: ret i32 1 // // +// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// // CHECK-LABEL: @_Z3barv( // CHECK-NEXT: entry: // CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 @@ -250,26 +270,6 @@ int bar() { // CHECK-NEXT: ret ptr @_Z3foov.default // // -// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: ret i32 1 -// -// -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 -// -// -// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 0 -// -// // CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -312,16 +312,16 @@ int bar() { // CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default // //. -// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } -// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } // CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" } // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } //. diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 47dbeca206a9..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -483,14 +483,16 @@ void just_fine(void) {} __arm_locally_streaming __attribute__((target_version("sme2"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} +// expected-error@-1 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} +// expected-cpp-error@-2 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} __attribute__((target_version("default"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} void fmv_caller() { cannot_work_version(); cannot_work_clones(); just_fine(); - just_fine_locally_streaming(); + incompatible_locally_streaming(); } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index e2940c434c2f..cd5be459456e 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -68,13 +68,15 @@ int __attribute__((target_version(""))) unsup1(void) { return 1; } void __attribute__((target_version("crc32"))) unsup2(void) {} void __attribute__((target_version("default+fp16"))) koo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("default+default+default"))) loo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} //expected-error@+2 {{redefinition of 'redef'}} //expected-note@-2 {{previous definition is here}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} -int __attribute__((target_version("sm4"))) def(void); +int def(void); void __attribute__((target_version("dit"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); @@ -83,7 +85,6 @@ int bar() { // expected-error@+2 {{reference to overloaded function could not be resolved; did you mean to call it?}} // expected-note@-3 {{possible target for call}} ovl++; - // expected-error@+1 {{no matching function for call to 'nodef'}} nodef(); return def(); } @@ -92,8 +93,6 @@ int __attribute__((target_version("sha1"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -// expected-note@+1 {{function multiversioning caused by this declaration}} -int __attribute__((target_version("fcma"))) prot(); int __attribute__((target_version("pmull"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} @@ -104,6 +103,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +// expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } int cargs() { return unspec_args(); } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index 0bd710c4e282..b3385f043590 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -9,7 +9,6 @@ void __attribute__((target_version("rcpc3"))) no_def(void); void __attribute__((target_version("mops"))) no_def(void); void __attribute__((target_version("rdma"))) no_def(void); -// expected-error@+1 {{no matching function for call to 'no_def'}} void foo(void) { no_def(); } constexpr int __attribute__((target_version("sve2"))) diff_const(void) { return 1; } @@ -41,6 +40,7 @@ inline int __attribute__((target_version("sme"))) diff_inline(void) { return 1; int __attribute__((target_version("fp16"))) diff_inline(void) { return 2; } inline int __attribute__((target_version("sme"))) diff_inline1(void) { return 1; } +//expected-error@+1 {{multiversioned function declaration has a different inline specification}} int __attribute__((target_version("default"))) diff_inline1(void) { return 2; } int __attribute__((target_version("fcma"))) diff_type1(void) { return 1; } @@ -59,8 +59,7 @@ int __attribute__((target_version("sve2-sha3"))) diff_type3(void) noexcept(true) template int __attribute__((target_version("default"))) temp(T) { return 1; } template int __attribute__((target_version("simd"))) temp1(T) { return 1; } -// expected-error@+1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} -template int __attribute__((target_version("sha3"))) temp1(T) { return 2; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} extern "C" { int __attribute__((target_version("aes"))) extc(void) { return 1; } @@ -70,17 +69,23 @@ int __attribute__((target_version("lse"))) extc(void) { return 1; } auto __attribute__((target_version("default"))) ret1(void) { return 1; } auto __attribute__((target_version("dpb"))) ret2(void) { return 1; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deduced return types}} auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} __attribute__((target_version("sve-i8mm"))) ~Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support defaulted functions}} Cls &__attribute__((target_version("ssbs"))) operator=(Cls &&) = delete; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deleted functions}} virtual void __attribute__((target_version("default"))) vfunc(); virtual void __attribute__((target_version("sm4"))) vfunc1(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support virtual functions}} }; __attribute__((target_version("sha3"))) void Decl(); -- GitLab From 7434a6b96c68b6835a6de6e98199dcfc35981dd5 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 25 Mar 2024 10:50:08 +0100 Subject: [PATCH 074/879] [TableGen] Fix Linker Errors Fix linker errors after landing fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 --- llvm/utils/TableGen/Basic/CMakeLists.txt | 2 +- llvm/utils/TableGen/CMakeLists.txt | 2 +- llvm/utils/TableGen/Common/CMakeLists.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt index f2927d05c175..5a899e3b7c80 100644 --- a/llvm/utils/TableGen/Basic/CMakeLists.txt +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -8,7 +8,7 @@ set(LLVM_LINK_COMPONENTS TableGen ) -add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL +add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL CodeGenIntrinsics.cpp SDNodeProperties.cpp ) diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 14690329cabf..577aeded4be7 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -75,10 +75,10 @@ add_tablegen(llvm-tblgen LLVM X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp + $ $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt index 491d9bd2949d..0e985a083247 100644 --- a/llvm/utils/TableGen/Common/CMakeLists.txt +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -39,7 +39,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL vt_gen ) set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") -target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) # Users may include its headers as "Common/*.h" target_include_directories(LLVMTableGenCommon -- GitLab From 336bdf1a255571f8d894e8befe4be7e9141f7541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Mon, 25 Mar 2024 11:52:29 +0200 Subject: [PATCH 075/879] [verify-uselistorder] Fix a typo, fix linking This fixes a typo from 2ef612050844355906e4b67d892a00bbb58c41d6, which broke the build with errors like: ld.lld: error: undefined symbol: LoadBitcodeIntoNewDbgInforFormat >>> referenced by verify-uselistorder.cpp >>> tools/verify-uselistorder/CMakeFiles/verify-uselistorder.dir/verify-uselistorder.cpp.o:(main) >>> did you mean: LoadBitcodeIntoNewDbgInfoFormat >>> defined in: lib/libLLVMBitReader.a(BitcodeReader.cpp.o) collect2: error: ld returned 1 exit status --- llvm/tools/verify-uselistorder/verify-uselistorder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index 8b299c394e4e..cb07dede1d13 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,7 +68,7 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); -extern cl::opt LoadBitcodeIntoNewDbgInforFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -546,8 +546,8 @@ int main(int argc, char **argv) { "llvm tool to verify use-list order\n"); // Do not load bitcode into the new debug info format by default. - if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) - LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_FALSE; LLVMContext Context; SMDiagnostic Err; -- GitLab From dbfc38ed6b3f2a9be0b1a86b2a074aad69eb58a6 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Mon, 25 Mar 2024 18:57:53 +0900 Subject: [PATCH 076/879] [mlir][bufferization] Add `BufferOriginAnalysis` (#86461) This commit adds the `BufferOriginAnalysis`, which can be queried to check if two buffer SSA values originate from the same allocation. This new analysis is used in the buffer deallocation pass to fold away or simplify `bufferization.dealloc` ops more aggressively. The `BufferOriginAnalysis` is based on the `BufferViewFlowAnalysis`, which collects buffer SSA value "same buffer" dependencies. E.g., given IR such as: ``` %0 = memref.alloc() %1 = memref.subview %0 %2 = memref.subview %1 ``` The `BufferViewFlowAnalysis` will report the following "reverse" dependencies (`resolveReverse`) for `%2`: {`%2`, `%1`, `%0`}. I.e., all buffer SSA values in the reverse use-def chain that originate from the same allocation as `%2`. The `BufferOriginAnalysis` is built on top of that. It handles only simple cases at the moment and may conservatively return "unknown" around certain IR with branches, memref globals and function arguments. This analysis enables additional simplifications during `-buffer-deallocation-simplification`. In particular, "regular" scf.for loop nests, that yield buffers (or reallocations thereof) in the same order as they appear in the iter_args, are now handled much more efficiently. Such IR patterns are generated by the sparse compiler. --- .../Bufferization/IR/BufferizationOps.td | 1 + .../Transforms/BufferViewFlowAnalysis.h | 36 ++++ .../BufferDeallocationSimplification.cpp | 80 ++++----- .../Transforms/BufferViewFlowAnalysis.cpp | 160 ++++++++++++++++-- .../dealloc-loops.mlir | 86 ++++++++++ .../buffer-deallocation-simplification.mlir | 14 +- 6 files changed, 321 insertions(+), 56 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td index 9dc6afcaab31..4f609ddff9a4 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td @@ -10,6 +10,7 @@ #define BUFFERIZATION_OPS include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td" +include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizationBase.td" include "mlir/Interfaces/DestinationStyleOpInterface.td" diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 9e43265c5dfe..4015231c845d 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -53,6 +53,7 @@ public: /// /// Results in resolve(B) returning {B, C} ValueSetT resolve(Value value) const; + ValueSetT resolveReverse(Value value) const; /// Removes the given values from all alias sets. void remove(const SetVector &aliasValues); @@ -73,11 +74,46 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + ValueMapT reverseDependencies; /// A set of all SSA values that may be terminal buffers. DenseSet terminals; }; +/// An is-same-buffer analysis that checks if two SSA values belong to the same +/// buffer allocation or not. +class BufferOriginAnalysis { +public: + BufferOriginAnalysis(Operation *op); + + /// Return "true" if `v1` and `v2` originate from the same buffer allocation. + /// Return "false" if `v1` and `v2` originate from different allocations. + /// Return "nullopt" if we do not know for sure. + /// + /// Example 1: isSameAllocation(%0, %1) == true + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.subview %0 + /// ``` + /// + /// Example 2: isSameAllocation(%0, %1) == false + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// ``` + /// + /// Example 3: isSameAllocation(%0, %2) == nullopt + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// %2 = arith.select %c, %0, %1 + /// ``` + std::optional isSameAllocation(Value v1, Value v2); + +private: + BufferViewFlowAnalysis analysis; +}; + } // namespace mlir #endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERVIEWFLOWANALYSIS_H diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp index e30779868b47..954485cfede3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Analysis/AliasAnalysis.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" #include "mlir/Dialect/Bufferization/Transforms/Passes.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" @@ -34,6 +34,14 @@ using namespace mlir::bufferization; // Helpers //===----------------------------------------------------------------------===// +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, ValueRange memrefs, ValueRange conditions, @@ -49,14 +57,6 @@ static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, return success(); } -/// Given a memref value, return the "base" value by skipping over all -/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. -static Value getViewBase(Value value) { - while (auto viewLikeOp = value.getDefiningOp()) - value = viewLikeOp.getViewSource(); - return value; -} - /// Return "true" if the given values are guaranteed to be different (and /// non-aliasing) allocations based on the fact that one value is the result /// of an allocation and the other value is a block argument of a parent block. @@ -80,12 +80,14 @@ static bool distinctAllocAndBlockArgument(Value v1, Value v2) { /// Checks if `memref` may potentially alias a MemRef in `otherList`. It is /// often a requirement of optimization patterns that there cannot be any /// aliasing memref in order to perform the desired simplification. -static bool potentiallyAliasesMemref(AliasAnalysis &analysis, +static bool potentiallyAliasesMemref(BufferOriginAnalysis &analysis, ValueRange otherList, Value memref) { for (auto other : otherList) { if (distinctAllocAndBlockArgument(other, memref)) continue; - if (!analysis.alias(other, memref).isNo()) + std::optional analysisResult = + analysis.isSameAllocation(other, memref); + if (!analysisResult.has_value() || analysisResult == true) return true; } return false; @@ -129,8 +131,8 @@ namespace { struct RemoveDeallocMemrefsContainedInRetained : public OpRewritePattern { RemoveDeallocMemrefsContainedInRetained(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} /// The passed 'memref' must not have a may-alias relation to any retained /// memref, and at least one must-alias relation. If there is no must-aliasing @@ -147,10 +149,11 @@ struct RemoveDeallocMemrefsContainedInRetained // deallocated in some situations and can thus not be dropped). bool atLeastOneMustAlias = false; for (Value retained : deallocOp.getRetained()) { - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMay()) + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (!analysisResult.has_value()) return failure(); - if (analysisResult.isMust() || analysisResult.isPartial()) + if (analysisResult == true) atLeastOneMustAlias = true; } if (!atLeastOneMustAlias) @@ -161,8 +164,9 @@ struct RemoveDeallocMemrefsContainedInRetained // we can remove that operand later on. for (auto [i, retained] : llvm::enumerate(deallocOp.getRetained())) { Value updatedCondition = deallocOp.getUpdatedConditions()[i]; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { auto disjunction = rewriter.create( deallocOp.getLoc(), updatedCondition, cond); rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(), @@ -206,7 +210,7 @@ struct RemoveDeallocMemrefsContainedInRetained } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Remove memrefs from the `retained` list which are guaranteed to not alias @@ -228,15 +232,15 @@ private: struct RemoveRetainedMemrefsGuaranteedToNotAlias : public OpRewritePattern { RemoveRetainedMemrefsGuaranteedToNotAlias(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { SmallVector newRetainedMemrefs, replacements; for (auto retainedMemref : deallocOp.getRetained()) { - if (potentiallyAliasesMemref(aliasAnalysis, deallocOp.getMemrefs(), + if (potentiallyAliasesMemref(analysis, deallocOp.getMemrefs(), retainedMemref)) { newRetainedMemrefs.push_back(retainedMemref); replacements.push_back({}); @@ -264,7 +268,7 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Split off memrefs to separate dealloc operations to reduce the number of @@ -297,8 +301,8 @@ private: struct SplitDeallocWhenNotAliasingAnyOther : public OpRewritePattern { SplitDeallocWhenNotAliasingAnyOther(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -314,7 +318,7 @@ struct SplitDeallocWhenNotAliasingAnyOther SmallVector otherMemrefs(deallocOp.getMemrefs()); otherMemrefs.erase(otherMemrefs.begin() + i); // Check if `memref` can split off into a separate bufferization.dealloc. - if (potentiallyAliasesMemref(aliasAnalysis, otherMemrefs, memref)) { + if (potentiallyAliasesMemref(analysis, otherMemrefs, memref)) { // `memref` alias with other memrefs, do not split off. remainingMemrefs.push_back(memref); remainingConditions.push_back(cond); @@ -352,7 +356,7 @@ struct SplitDeallocWhenNotAliasingAnyOther } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Check for every retained memref if a must-aliasing memref exists in the @@ -381,8 +385,8 @@ private: struct RetainedMemrefAliasingAlwaysDeallocatedMemref : public OpRewritePattern { RetainedMemrefAliasingAlwaysDeallocatedMemref(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -396,8 +400,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!matchPattern(cond, m_One())) continue; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -411,10 +416,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!extractOp) continue; - AliasResult extractAnalysisResult = - aliasAnalysis.alias(retained, extractOp.getOperand()); - if (extractAnalysisResult.isMust() || - extractAnalysisResult.isPartial()) { + std::optional extractAnalysisResult = + analysis.isSameAllocation(retained, extractOp.getOperand()); + if (extractAnalysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -434,7 +438,7 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; } // namespace @@ -452,13 +456,13 @@ struct BufferDeallocationSimplificationPass : public bufferization::impl::BufferDeallocationSimplificationBase< BufferDeallocationSimplificationPass> { void runOnOperation() override { - AliasAnalysis &aliasAnalysis = getAnalysis(); + BufferOriginAnalysis analysis(getOperation()); RewritePatternSet patterns(&getContext()); patterns.add(&getContext(), - aliasAnalysis); + analysis); populateDeallocOpCanonicalizationPatterns(patterns, &getContext()); if (failed( diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 9a36057425f3..72f47b8b468e 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -19,22 +19,23 @@ using namespace mlir; using namespace mlir::bufferization; +//===----------------------------------------------------------------------===// +// BufferViewFlowAnalysis +//===----------------------------------------------------------------------===// + /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } -/// Find all immediate and indirect dependent buffers this value could -/// potentially have. Note that the resulting set will also contain the value -/// provided as it is a dependent alias of itself. -BufferViewFlowAnalysis::ValueSetT -BufferViewFlowAnalysis::resolve(Value rootValue) const { - ValueSetT result; +static BufferViewFlowAnalysis::ValueSetT +resolveValues(const BufferViewFlowAnalysis::ValueMapT &map, Value value) { + BufferViewFlowAnalysis::ValueSetT result; SmallVector queue; - queue.push_back(rootValue); + queue.push_back(value); while (!queue.empty()) { Value currentValue = queue.pop_back_val(); if (result.insert(currentValue).second) { - auto it = dependencies.find(currentValue); - if (it != dependencies.end()) { + auto it = map.find(currentValue); + if (it != map.end()) { for (Value aliasValue : it->second) queue.push_back(aliasValue); } @@ -43,6 +44,19 @@ BufferViewFlowAnalysis::resolve(Value rootValue) const { return result; } +/// Find all immediate and indirect dependent buffers this value could +/// potentially have. Note that the resulting set will also contain the value +/// provided as it is a dependent alias of itself. +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolve(Value rootValue) const { + return resolveValues(dependencies, rootValue); +} + +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolveReverse(Value rootValue) const { + return resolveValues(reverseDependencies, rootValue); +} + /// Removes the given values from all alias sets. void BufferViewFlowAnalysis::remove(const SetVector &aliasValues) { for (auto &entry : dependencies) @@ -69,8 +83,10 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip_equal(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) { this->dependencies[value].insert(dep); + this->reverseDependencies[dep].insert(value); + } }; // Mark all buffer results and buffer region entry block arguments of the @@ -188,3 +204,127 @@ bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { assert(isa(value.getType()) && "expected memref"); return terminals.contains(value); } + +//===----------------------------------------------------------------------===// +// BufferOriginAnalysis +//===----------------------------------------------------------------------===// + +/// Return "true" if the given value is the result of a memory allocation. +static bool hasAllocateSideEffect(Value v) { + Operation *op = v.getDefiningOp(); + if (!op) + return false; + return hasEffect(op, v); +} + +/// Return "true" if the given value is a function block argument. +static bool isFunctionArgument(Value v) { + auto bbArg = dyn_cast(v); + if (!bbArg) + return false; + Block *b = bbArg.getOwner(); + auto funcOp = dyn_cast(b->getParentOp()); + if (!funcOp) + return false; + return bbArg.getOwner() == &funcOp.getFunctionBody().front(); +} + +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + +BufferOriginAnalysis::BufferOriginAnalysis(Operation *op) : analysis(op) {} + +std::optional BufferOriginAnalysis::isSameAllocation(Value v1, Value v2) { + assert(isa(v1.getType()) && "expected buffer"); + assert(isa(v2.getType()) && "expected buffer"); + + // Skip over all view-like ops. + v1 = getViewBase(v1); + v2 = getViewBase(v2); + + // Fast path: If both buffers are the same SSA value, we can be sure that + // they originate from the same allocation. + if (v1 == v2) + return true; + + // Compute the SSA values from which the buffers `v1` and `v2` originate. + SmallPtrSet origin1 = analysis.resolveReverse(v1); + SmallPtrSet origin2 = analysis.resolveReverse(v2); + + // Originating buffers are "terminal" if they could not be traced back any + // further by the `BufferViewFlowAnalysis`. Examples of terminal buffers: + // - function block arguments + // - values defined by allocation ops such as "memref.alloc" + // - values defined by ops that are unknown to the buffer view flow analysis + // - values that are marked as "terminal" in the `BufferViewFlowOpInterface` + SmallPtrSet terminal1, terminal2; + + // While gathering terminal buffers, keep track of whether all terminal + // buffers are newly allocated buffer or function entry arguments. + bool allAllocs1 = true, allAllocs2 = true; + bool allAllocsOrFuncEntryArgs1 = true, allAllocsOrFuncEntryArgs2 = true; + + // Helper function that gathers terminal buffers among `origin`. + auto gatherTerminalBuffers = [this](const SmallPtrSet &origin, + SmallPtrSet &terminal, + bool &allAllocs, + bool &allAllocsOrFuncEntryArgs) { + for (Value v : origin) { + if (isa(v.getType()) && analysis.mayBeTerminalBuffer(v)) { + terminal.insert(v); + allAllocs &= hasAllocateSideEffect(v); + allAllocsOrFuncEntryArgs &= + isFunctionArgument(v) || hasAllocateSideEffect(v); + } + } + assert(!terminal.empty() && "expected non-empty terminal set"); + }; + + // Gather terminal buffers for `v1` and `v2`. + gatherTerminalBuffers(origin1, terminal1, allAllocs1, + allAllocsOrFuncEntryArgs1); + gatherTerminalBuffers(origin2, terminal2, allAllocs2, + allAllocsOrFuncEntryArgs2); + + // If both `v1` and `v2` have a single matching terminal buffer, they are + // guaranteed to originate from the same buffer allocation. + if (llvm::hasSingleElement(terminal1) && llvm::hasSingleElement(terminal2) && + *terminal1.begin() == *terminal2.begin()) + return true; + + // At least one of the two values has multiple terminals. + + // Check if there is overlap between the terminal buffers of `v1` and `v2`. + bool distinctTerminalSets = true; + for (Value v : terminal1) + distinctTerminalSets &= !terminal2.contains(v); + // If there is overlap between the terminal buffers of `v1` and `v2`, we + // cannot make an accurate decision without further analysis. + if (!distinctTerminalSets) + return std::nullopt; + + // If `v1` originates from only allocs, and `v2` is guaranteed to originate + // from different allocations (that is guaranteed if `v2` originates from + // only distinct allocs or function entry arguments), we can be sure that + // `v1` and `v2` originate from different allocations. The same argument can + // be made when swapping `v1` and `v2`. + bool isolatedAlloc1 = allAllocs1 && (allAllocs2 || allAllocsOrFuncEntryArgs2); + bool isolatedAlloc2 = (allAllocs1 || allAllocsOrFuncEntryArgs1) && allAllocs2; + if (isolatedAlloc1 || isolatedAlloc2) + return false; + + // Otherwise: We do not know whether `v1` and `v2` originate from the same + // allocation or not. + // TODO: Function arguments are currently handled conservatively. We assume + // that they could be the same allocation. + // TODO: Terminals other than allocations and function arguments are + // currently handled conservatively. We assume that they could be the same + // allocation. E.g., we currently return "nullopt" for values that originate + // from different "memref.get_global" ops (with different symbols). + return std::nullopt; +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir new file mode 100644 index 000000000000..53b28c3aab6f --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir @@ -0,0 +1,86 @@ +// RUN: mlir-opt %s -expand-realloc="emit-deallocs=false" -ownership-based-buffer-deallocation="private-function-dynamic-ownership=true" -canonicalize -buffer-deallocation-simplification | FileCheck %s + +// A function that reallocates two buffer inside of a loop. The simplification +// pass should be able to figure out that the iter_args are always originating +// from different allocations. IR like this one appears in the sparse compiler. + +// CHECK-LABEL: func private @loop_with_realloc( +func.func private @loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + // CHECK: %[[m2:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m2 = memref.realloc %arg0(%s2) : memref to memref + // CHECK: %[[m3:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m3 = memref.realloc %arg1(%s2) : memref to memref + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg0]] + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg1]] + // CHECK: %[[d0:.*]] = bufferization.dealloc (%[[base0]] : memref) if (%[[o0]]) retain (%[[m2]]#0 : memref) + // CHECK: %[[d1:.*]] = bufferization.dealloc (%[[base1]] : memref) if (%[[o1]]) retain (%[[m3]]#0 : memref) + // CHECK-DAG: %[[o2:.*]] = arith.ori %[[d0]], %[[m2]]#1 + // CHECK-DAG: %[[o3:.*]] = arith.ori %[[d1]], %[[m3]]#1 + // CHECK: scf.yield %[[m2]]#0, %[[m3]]#0, %[[o2]], %[[o3]] + scf.yield %m2, %m3 : memref, memref + } + + // CHECK: %[[d2:.*]] = bufferization.dealloc (%[[m0]] : memref) if (%[[true]]) retain (%[[r]]#0 : memref) + // CHECK: %[[d3:.*]] = bufferization.dealloc (%[[m1]] : memref) if (%[[true]]) retain (%[[r]]#1 : memref) + // CHECK-DAG: %[[or0:.*]] = arith.ori %[[d2]], %[[r]]#2 + // CHECK-DAG: %[[or1:.*]] = arith.ori %[[d3]], %[[r]]#3 + // CHECK: return %[[r]]#0, %[[r]]#1, %[[or0]], %[[or1]] + return %r0, %r1 : memref, memref +} + +// ----- + +// The yielded values of the loop are swapped. Therefore, the +// bufferization.dealloc before the func.return can now longer be split, +// because %r0 could originate from either %m0 and %m1 (same for %r1). + +// CHECK-LABEL: func private @swapping_loop_with_realloc( +func.func private @swapping_loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + %m2 = memref.realloc %arg0(%s2) : memref to memref + %m3 = memref.realloc %arg1(%s2) : memref to memref + scf.yield %m3, %m2 : memref, memref + } + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#0 + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#1 + // CHECK: %[[d:.*]]:2 = bufferization.dealloc (%[[m0]], %[[m1]], %[[base0]], %[[base1]] : {{.*}}) if (%[[true]], %[[true]], %[[r]]#2, %[[r]]#3) retain (%[[r]]#0, %[[r]]#1 : {{.*}}) + // CHECK: return %[[r]]#0, %[[r]]#1, %[[d]]#0, %[[d]]#1 + return %r0, %r1 : memref, memref +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir index eee69acbe821..b40a17cf800b 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir @@ -92,15 +92,13 @@ func.func @dealloc_split_when_no_other_aliasing(%arg0: i1, %arg1: memref<2xi32>, // CHECK-NEXT: [[ALLOC0:%.+]] = memref.alloc( // CHECK-NEXT: [[ALLOC1:%.+]] = memref.alloc( // CHECK-NEXT: [[V0:%.+]] = arith.select{{.*}}[[ALLOC0]], [[ALLOC1]] : -// COM: there is only one value in the retained list because the -// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here and -// COM: removes %arg1 from the list. In the second dealloc, this does not apply -// COM: because function arguments are assumed potentially alias (even if the -// COM: types don't exactly match). +// COM: there is only one value in the retained lists because the +// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here: +// COM: - %alloc is guaranteed to not alias with %arg1. +// COM: - %arg2 is guaranteed to not alias with %0. // CHECK-NEXT: [[V1:%.+]] = bufferization.dealloc ([[ALLOC0]] : memref<2xi32>) if ([[ARG0]]) retain ([[V0]] : memref<2xi32>) -// CHECK-NEXT: [[V2:%.+]]:2 = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]], [[V0]] : memref<2xi32>, memref<2xi32>) -// CHECK-NEXT: [[V3:%.+]] = arith.ori [[V1]], [[V2]]#1 -// CHECK-NEXT: return [[V2]]#0, [[V3]] : +// CHECK-NEXT: [[V2:%.+]] = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]] : memref<2xi32>) +// CHECK-NEXT: return [[V2]], [[V1]] : // ----- -- GitLab From 94a550dab26c4b30a187dd8e3ce431e0f915923b Mon Sep 17 00:00:00 2001 From: Mariusz Sikora Date: Mon, 25 Mar 2024 11:00:59 +0100 Subject: [PATCH 077/879] [AMDGPU][NFC] Rename Feature GFX11FullVGPRs to 1_5xVGPRs (#86468) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 14 ++++++++------ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index c877658cd38e..37dcfef3b2a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -831,10 +831,12 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard", "Does not need SW waitstates" >; -def FeatureGFX11FullVGPRs : SubtargetFeature<"gfx11-full-vgprs", - "HasGFX11FullVGPRs", +// Allocate 1536 VGPRs for wave32 and 768 VGPRs for wave64 +// with allocation granularity 24 for wave32 and 12 for wave64 +def Feature1_5xVGPRs : SubtargetFeature<"allocate1_5xvgprs", + "Has1_5xVGPRs", "true", - "GFX11 with 50% more physical VGPRs and 50% larger allocation granule than GFX10" + "Has 50% more physical VGPRs and 50% larger allocation granule" >; @@ -1491,12 +1493,12 @@ def FeatureISAVersion11_0_Common : FeatureSet< def FeatureISAVersion11_0_0 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs, + [Feature1_5xVGPRs, FeatureUserSGPRInit16Bug])>; def FeatureISAVersion11_0_1 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs])>; + [Feature1_5xVGPRs])>; def FeatureISAVersion11_0_2 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, @@ -1517,7 +1519,7 @@ def FeatureISAVersion11_5_1 : FeatureSet< [FeatureSALUFloatInsts, FeatureDPPSrc1SGPR, FeatureVGPRSingleUseHintInsts, - FeatureGFX11FullVGPRs])>; + Feature1_5xVGPRs])>; def FeatureISAVersion12 : FeatureSet< [FeatureGFX12, diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index ca51da659c33..4da10beabe31 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -223,7 +223,7 @@ protected: bool HasImageStoreD16Bug = false; bool HasImageGather4D16Bug = false; bool HasMSAALoadDstSelBug = false; - bool HasGFX11FullVGPRs = false; + bool Has1_5xVGPRs = false; bool HasMADIntraFwdBug = false; bool HasVOPDInsts = false; bool HasVALUTransUseHazard = false; @@ -1202,7 +1202,7 @@ public: /// target. bool hasNullExportTarget() const { return !GFX11Insts; } - bool hasGFX11FullVGPRs() const { return HasGFX11FullVGPRs; } + bool has1_5xVGPRs() const { return Has1_5xVGPRs; } bool hasVOPDInsts() const { return HasVOPDInsts; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index a90dc32d396f..7bb84d78442b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1087,7 +1087,7 @@ unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, *EnableWavefrontSize32 : STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 24 : 12; if (hasGFX10_3Insts(*STI)) @@ -1114,7 +1114,7 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { if (!isGFX10Plus(*STI)) return 256; bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 1536 : 768; return IsWave32 ? 1024 : 512; } -- GitLab From 37785fedabd8fa752129ef5bac3462311af91c35 Mon Sep 17 00:00:00 2001 From: Discookie Date: Mon, 25 Mar 2024 10:08:56 +0000 Subject: [PATCH 078/879] [clang][analyzer] Bring cplusplus.ArrayDelete out of alpha (#83985) The checker finds a type of undefined behavior, where if the type of a pointer to an object-array is different from the objects' underlying type, calling `delete[]` is undefined, as the size of the two objects might be different. The checker has been in alpha for a while now, it is a simple checker that causes no crashes, and considering the severity of the issue, it has a low result-count on open-source projects (in my last test-run on my usual projects, it had 0 results). This commit cleans up the documentation and adds docs for the limitation related to tracking through references, in addition to moving it to `cplusplus`. --------- Co-authored-by: Balazs Benics Co-authored-by: whisperity --- clang/docs/analyzer/checkers.rst | 69 ++++++++++++------- .../clang/StaticAnalyzer/Checkers/Checkers.td | 10 +-- .../Checkers/CXXDeleteChecker.cpp | 4 +- clang/test/Analysis/ArrayDelete.cpp | 2 +- clang/www/analyzer/alpha_checks.html | 20 ------ clang/www/analyzer/available_checks.html | 27 ++++++++ 6 files changed, 80 insertions(+), 52 deletions(-) diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst index fe2115149142..66da1c7b35f2 100644 --- a/clang/docs/analyzer/checkers.rst +++ b/clang/docs/analyzer/checkers.rst @@ -340,6 +340,51 @@ cplusplus C++ Checkers. +.. _cplusplus-ArrayDelete: + +cplusplus.ArrayDelete (C++) +""""""""""""""""""""""""""" + +Reports destructions of arrays of polymorphic objects that are destructed as +their base class. If the dynamic type of the array is different from its static +type, calling `delete[]` is undefined. + +This checker corresponds to the SEI CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. + +.. code-block:: cpp + + class Base { + public: + virtual ~Base() {} + }; + class Derived : public Base {}; + + Base *create() { + Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here + return x; + } + + void foo() { + Base *x = create(); + delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + +**Limitations** + +The checker does not emit note tags when casting to and from reference types, +even though the pointer values are tracked across references. + +.. code-block:: cpp + + void foo() { + Derived *d = new Derived[10]; + Derived &dref = *d; + + Base &bref = static_cast(dref); // no note + Base *b = &bref; + delete[] b; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + .. _cplusplus-InnerPointer: cplusplus.InnerPointer (C++) @@ -2139,30 +2184,6 @@ Either the comparison is useless or there is division by zero. alpha.cplusplus ^^^^^^^^^^^^^^^ -.. _alpha-cplusplus-ArrayDelete: - -alpha.cplusplus.ArrayDelete (C++) -""""""""""""""""""""""""""""""""" -Reports destructions of arrays of polymorphic objects that are destructed as their base class. -This checker corresponds to the CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. - -.. code-block:: cpp - - class Base { - virtual ~Base() {} - }; - class Derived : public Base {} - - Base *create() { - Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here - return x; - } - - void foo() { - Base *x = create(); - delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined - } - .. _alpha-cplusplus-DeleteWithNonVirtualDtor: alpha.cplusplus.DeleteWithNonVirtualDtor (C++) diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td index 686e5e99f4a6..bf46766d44b3 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td @@ -622,6 +622,11 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">, let ParentPackage = Cplusplus in { +def ArrayDeleteChecker : Checker<"ArrayDelete">, + HelpText<"Reports destructions of arrays of polymorphic objects that are " + "destructed as their base class.">, + Documentation; + def InnerPointerChecker : Checker<"InnerPointer">, HelpText<"Check for inner pointers of C++ containers used after " "re/deallocation">, @@ -777,11 +782,6 @@ def ContainerModeling : Checker<"ContainerModeling">, Documentation, Hidden; -def CXXArrayDeleteChecker : Checker<"ArrayDelete">, - HelpText<"Reports destructions of arrays of polymorphic objects that are " - "destructed as their base class.">, - Documentation; - def DeleteWithNonVirtualDtorChecker : Checker<"DeleteWithNonVirtualDtor">, HelpText<"Reports destructions of polymorphic objects with a non-virtual " "destructor in their base class">, diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp index b4dee1e300e8..1b1226a7f1a7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp @@ -220,11 +220,11 @@ CXXDeleteChecker::PtrCastVisitor::VisitNode(const ExplodedNode *N, /*addPosRange=*/true); } -void ento::registerCXXArrayDeleteChecker(CheckerManager &mgr) { +void ento::registerArrayDeleteChecker(CheckerManager &mgr) { mgr.registerChecker(); } -bool ento::shouldRegisterCXXArrayDeleteChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterArrayDeleteChecker(const CheckerManager &mgr) { return true; } diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp index 3b8d49552376..6887e0a35fb8 100644 --- a/clang/test/Analysis/ArrayDelete.cpp +++ b/clang/test/Analysis/ArrayDelete.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s +// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s struct Base { virtual ~Base() = default; diff --git a/clang/www/analyzer/alpha_checks.html b/clang/www/analyzer/alpha_checks.html index 7bbe4a20288f..f040d1957b0f 100644 --- a/clang/www/analyzer/alpha_checks.html +++ b/clang/www/analyzer/alpha_checks.html @@ -307,26 +307,6 @@ void test(int x) { - - - -